From 88435edc206053fed119a217eb18769ba3581772 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Thu, 28 May 2026 11:24:45 +0300 Subject: [PATCH] chore: add github migration foundation --- .env.example | 17 + .github/ISSUE_TEMPLATE/bug_report.yml | 52 + .github/ISSUE_TEMPLATE/feature_request.yml | 44 + .github/PULL_REQUEST_TEMPLATE.md | 20 + .github/workflows/ci.yml | 29 + .gitignore | 49 + AGENTS.md | 197 +++ CHANGELOG.md | 14 + CODEX.md | 150 ++ CONTRIBUTING.md | 58 + CURRENT_SCOPE.md | 59 + DATA_CONTRACTS.md | 55 + EXPORT_MANIFEST.md | 44 + PHASE2_GITHUB_TRANSFER_INSTRUCTIONS.md | 42 + PUBLICATION_AUDIT.md | 65 + PUBLICATION_CHECKLIST.md | 15 + README.md | 356 ++++- RUNBOOK.md | 109 ++ SPEC.md | 73 + SPEC_AGENTS_CLEANUP.md | 71 + SPEC_chatgpt_compact_kb.md | 117 ++ SPEC_synthesis_layer.md | 112 ++ SPEC_transcript_to_kb_pipeline.md | 1271 +++++++++++++++++ SPEC_v1.md | 645 +++++++++ docs/migration/generated_artifacts_policy.md | 27 + docs/migration/github_transfer_plan.md | 57 + docs/migration/publication_scope.md | 85 ++ docs/migration/raw_input_policy.md | 34 + input/.gitkeep | 0 input/README.md | 12 + plan.md | 84 ++ prompts/clean_note_ru.md | 35 + prompts/kb_build_ru.md | 11 + prompts/source_card_ru.md | 35 + requirements.txt | 2 + scripts/build_chatgpt_compact_kb.py | 945 ++++++++++++ scripts/build_release_manifest.py | 304 ++++ scripts/build_synthesis_layer.py | 909 ++++++++++++ scripts/run_acceptance_gate.py | 96 ++ scripts/run_chunking.py | 23 + scripts/run_clean_note.py | 62 + scripts/run_deduplication.py | 57 + scripts/run_gemini_pipeline.py | 144 ++ scripts/run_inventory.py | 25 + scripts/run_judge.py | 50 + scripts/run_kb_build.py | 49 + scripts/run_managed_knowledge_factory.py | 73 + scripts/run_ollama_pipeline.py | 123 ++ scripts/run_promotion_gate.py | 45 + scripts/run_publish.py | 74 + scripts/run_retrieval_qa.py | 133 ++ scripts/run_source_card.py | 118 ++ scripts/search_kb.py | 62 + scripts/sync_transcript_clean.py | 53 + scripts/validate_card_passports.py | 64 + src/notes_to_kb/__init__.py | 1 + src/notes_to_kb/chunking.py | 129 ++ src/notes_to_kb/clean_note.py | 302 ++++ src/notes_to_kb/errors.py | 26 + src/notes_to_kb/governance.py | 513 +++++++ src/notes_to_kb/inventory.py | 181 +++ src/notes_to_kb/judge.py | 387 +++++ src/notes_to_kb/kb_build.py | 245 ++++ src/notes_to_kb/llm_client.py | 337 +++++ src/notes_to_kb/paths.py | 31 + src/notes_to_kb/publish.py | 628 ++++++++ src/notes_to_kb/search.py | 55 + src/notes_to_kb/source_card.py | 561 ++++++++ tasks.md | 69 + tests/conftest.py | 54 + tests/fixtures/raw/sample_01.md | 5 + tests/fixtures/raw/sample_01.txt | 3 + tests/fixtures/raw/sample_ambiguous_01.md | 2 + tests/fixtures/raw/sample_ambiguous_01.txt | 1 + tests/fixtures/raw/sample_ambiguous_01_alt.md | 2 + .../fixtures/raw/sample_duplicate_sections.md | 10 + .../raw/sample_duplicate_sections.txt | 1 + tests/fixtures/raw/sample_missing_md.txt | 1 + tests/fixtures/raw/sample_missing_txt.md | 2 + tests/fixtures/raw/sample_with_think.md | 5 + tests/fixtures/raw/sample_with_think.txt | 2 + tests/test_acceptance_check_mvp_9_1.py | 77 + tests/test_acceptance_check_mvp_9_2.py | 85 ++ tests/test_acceptance_check_mvp_9_4.py | 50 + tests/test_acceptance_gate_runtime.py | 71 + tests/test_card_passport_validation.py | 127 ++ tests/test_chunking.py | 46 + tests/test_clean_note.py | 236 +++ tests/test_consumer_qa_evidence.py | 77 + tests/test_consumer_qa_package.py | 63 + tests/test_consumer_qa_rerun_mvp_9_2.py | 66 + tests/test_consumer_qa_rerun_mvp_9_4.py | 81 ++ tests/test_deduplication_runtime.py | 49 + tests/test_end_to_end_managed_pipeline.py | 33 + tests/test_inventory.py | 36 + tests/test_judge.py | 153 ++ tests/test_kb_build.py | 144 ++ tests/test_kb_structure_revision.py | 85 ++ tests/test_kb_structure_revision_pass_2.py | 139 ++ tests/test_managed_knowledge_system.py | 156 ++ tests/test_manual_qa_runbook.py | 54 + tests/test_ollama_client.py | 305 ++++ tests/test_promotion_gate_runtime.py | 94 ++ tests/test_publish.py | 141 ++ ...test_publish_preserves_navigation_layer.py | 96 ++ tests/test_publish_release_package.py | 60 + tests/test_release_manifest_controls.py | 50 + tests/test_retrieval_qa_runtime.py | 40 + tests/test_runtime_confidence_controls.py | 20 + tests/test_search_cli_mvp_10.py | 94 ++ tests/test_source_card.py | 389 +++++ tests/test_sync_transcript_clean.py | 35 + 112 files changed, 14158 insertions(+), 2 deletions(-) create mode 100644 .env.example create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 AGENTS.md create mode 100644 CHANGELOG.md create mode 100644 CODEX.md create mode 100644 CONTRIBUTING.md create mode 100644 CURRENT_SCOPE.md create mode 100644 DATA_CONTRACTS.md create mode 100644 EXPORT_MANIFEST.md create mode 100644 PHASE2_GITHUB_TRANSFER_INSTRUCTIONS.md create mode 100644 PUBLICATION_AUDIT.md create mode 100644 PUBLICATION_CHECKLIST.md create mode 100644 RUNBOOK.md create mode 100644 SPEC.md create mode 100644 SPEC_AGENTS_CLEANUP.md create mode 100644 SPEC_chatgpt_compact_kb.md create mode 100644 SPEC_synthesis_layer.md create mode 100644 SPEC_transcript_to_kb_pipeline.md create mode 100644 SPEC_v1.md create mode 100644 docs/migration/generated_artifacts_policy.md create mode 100644 docs/migration/github_transfer_plan.md create mode 100644 docs/migration/publication_scope.md create mode 100644 docs/migration/raw_input_policy.md create mode 100644 input/.gitkeep create mode 100644 input/README.md create mode 100644 plan.md create mode 100644 prompts/clean_note_ru.md create mode 100644 prompts/kb_build_ru.md create mode 100644 prompts/source_card_ru.md create mode 100644 requirements.txt create mode 100644 scripts/build_chatgpt_compact_kb.py create mode 100644 scripts/build_release_manifest.py create mode 100644 scripts/build_synthesis_layer.py create mode 100644 scripts/run_acceptance_gate.py create mode 100644 scripts/run_chunking.py create mode 100644 scripts/run_clean_note.py create mode 100644 scripts/run_deduplication.py create mode 100755 scripts/run_gemini_pipeline.py create mode 100644 scripts/run_inventory.py create mode 100644 scripts/run_judge.py create mode 100644 scripts/run_kb_build.py create mode 100644 scripts/run_managed_knowledge_factory.py create mode 100755 scripts/run_ollama_pipeline.py create mode 100644 scripts/run_promotion_gate.py create mode 100644 scripts/run_publish.py create mode 100644 scripts/run_retrieval_qa.py create mode 100644 scripts/run_source_card.py create mode 100644 scripts/search_kb.py create mode 100644 scripts/sync_transcript_clean.py create mode 100644 scripts/validate_card_passports.py create mode 100644 src/notes_to_kb/__init__.py create mode 100644 src/notes_to_kb/chunking.py create mode 100644 src/notes_to_kb/clean_note.py create mode 100644 src/notes_to_kb/errors.py create mode 100644 src/notes_to_kb/governance.py create mode 100644 src/notes_to_kb/inventory.py create mode 100644 src/notes_to_kb/judge.py create mode 100644 src/notes_to_kb/kb_build.py create mode 100644 src/notes_to_kb/llm_client.py create mode 100644 src/notes_to_kb/paths.py create mode 100644 src/notes_to_kb/publish.py create mode 100644 src/notes_to_kb/search.py create mode 100644 src/notes_to_kb/source_card.py create mode 100644 tasks.md create mode 100644 tests/conftest.py create mode 100644 tests/fixtures/raw/sample_01.md create mode 100644 tests/fixtures/raw/sample_01.txt create mode 100644 tests/fixtures/raw/sample_ambiguous_01.md create mode 100644 tests/fixtures/raw/sample_ambiguous_01.txt create mode 100644 tests/fixtures/raw/sample_ambiguous_01_alt.md create mode 100644 tests/fixtures/raw/sample_duplicate_sections.md create mode 100644 tests/fixtures/raw/sample_duplicate_sections.txt create mode 100644 tests/fixtures/raw/sample_missing_md.txt create mode 100644 tests/fixtures/raw/sample_missing_txt.md create mode 100644 tests/fixtures/raw/sample_with_think.md create mode 100644 tests/fixtures/raw/sample_with_think.txt create mode 100644 tests/test_acceptance_check_mvp_9_1.py create mode 100644 tests/test_acceptance_check_mvp_9_2.py create mode 100644 tests/test_acceptance_check_mvp_9_4.py create mode 100644 tests/test_acceptance_gate_runtime.py create mode 100644 tests/test_card_passport_validation.py create mode 100644 tests/test_chunking.py create mode 100644 tests/test_clean_note.py create mode 100644 tests/test_consumer_qa_evidence.py create mode 100644 tests/test_consumer_qa_package.py create mode 100644 tests/test_consumer_qa_rerun_mvp_9_2.py create mode 100644 tests/test_consumer_qa_rerun_mvp_9_4.py create mode 100644 tests/test_deduplication_runtime.py create mode 100644 tests/test_end_to_end_managed_pipeline.py create mode 100644 tests/test_inventory.py create mode 100644 tests/test_judge.py create mode 100644 tests/test_kb_build.py create mode 100644 tests/test_kb_structure_revision.py create mode 100644 tests/test_kb_structure_revision_pass_2.py create mode 100644 tests/test_managed_knowledge_system.py create mode 100644 tests/test_manual_qa_runbook.py create mode 100644 tests/test_ollama_client.py create mode 100644 tests/test_promotion_gate_runtime.py create mode 100644 tests/test_publish.py create mode 100644 tests/test_publish_preserves_navigation_layer.py create mode 100644 tests/test_publish_release_package.py create mode 100644 tests/test_release_manifest_controls.py create mode 100644 tests/test_retrieval_qa_runtime.py create mode 100644 tests/test_runtime_confidence_controls.py create mode 100644 tests/test_search_cli_mvp_10.py create mode 100644 tests/test_source_card.py create mode 100644 tests/test_sync_transcript_clean.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..f12ef1b --- /dev/null +++ b/.env.example @@ -0,0 +1,17 @@ +# Copy this file to `.env` and fill only the keys you use. +# `.env` is ignored by git and must not be committed. + +# Required only for the Gemini route. +GEMINI_API_KEY=put_your_gemini_api_key_here + +# Optional Gemini settings. +GEMINI_SOURCE_CARD_MODEL=gemini-2.5-flash-lite +GEMINI_BASE_URL=https://generativelanguage.googleapis.com +GEMINI_TIMEOUT=600 +GEMINI_MAX_OUTPUT_TOKENS=4096 +GEMINI_THINKING_BUDGET=0 +GEMINI_MAX_RETRIES=3 + +# Optional local Ollama settings. +OLLAMA_BASE_URL=http://127.0.0.1:11434 +CLEAN_MODEL=devstral-small-2:24b-instruct-2512-q4_K_M diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..b030c05 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,52 @@ +name: Bug report +description: Report a reproducible problem. +title: "[Bug]: " +labels: ["bug"] +body: + - type: textarea + id: problem + attributes: + label: Problem + description: What is wrong? + validations: + required: true + - type: textarea + id: steps + attributes: + label: Steps to reproduce + description: List the exact commands or actions. + validations: + required: true + - type: textarea + id: expected + attributes: + label: Expected behavior + validations: + required: true + - type: textarea + id: actual + attributes: + label: Actual behavior + validations: + required: true + - type: textarea + id: environment + attributes: + label: Environment + description: OS, Python version, relevant dependency versions. + validations: + required: false + - type: textarea + id: logs + attributes: + label: Logs with secrets removed + description: Remove tokens, API keys, passwords, private paths, and raw transcript content. + validations: + required: false + - type: textarea + id: files + attributes: + label: Affected files + description: List known affected files or folders. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..5757a84 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,44 @@ +name: Feature request +description: Propose a scoped improvement. +title: "[Feature]: " +labels: ["enhancement"] +body: + - type: textarea + id: objective + attributes: + label: Objective + description: What should this change achieve? + validations: + required: true + - type: textarea + id: use_case + attributes: + label: Use case + description: Who needs this and why? + validations: + required: true + - type: textarea + id: proposed_behavior + attributes: + label: Proposed behavior + validations: + required: true + - type: textarea + id: acceptance + attributes: + label: Acceptance criteria + validations: + required: true + - type: textarea + id: risks + attributes: + label: Risks + description: Include privacy, data integrity, generated artifact, or compatibility risks. + validations: + required: false + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + validations: + required: false diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..de16c2c --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,20 @@ +## Summary + +## Scope + +## Files changed + +## Checks run + +## Publication safety +- [ ] No `.env` or secrets included +- [ ] No raw transcripts included +- [ ] No generated artifacts included unless approved +- [ ] `.env.example` is safe +- [ ] `.gitignore` still protects local/private files + +## Risks + +## Rollback + +## Acceptance status diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..42d7df9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,29 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + - name: Run tests + run: | + python -m pytest tests -q diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ab22c9a --- /dev/null +++ b/.gitignore @@ -0,0 +1,49 @@ +# Secrets / local env +.env +.env.* +!.env.example +*.key +*.pem +*.p12 + +# macOS / editor +.DS_Store +.idea/ +.vscode/ +*.swp + +# Python +__pycache__/ +*.py[cod] +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.venv/ +venv/ +env/ + +# Logs / temp +*.log +*.tmp +*.bak +tmp/ +temp/ + +# Local/private inputs +input/raw/ +input/private/ +input/local/ + +# Generated workspace/output artifacts +workspace/ +publish/ +publish_assets/ +release_snapshots/ +artifacts/ +runs/ +outputs/ +output/ + +# Local LLM / runtime caches +.ollama/ +.cache/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..e2f657c --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,197 @@ +# AGENTS.md + +Version: v03 +Status: active +Scope: repository-level agent instructions +Last updated: 2026-05-21 + +## 1. Active instruction priority + +For Codex work in this repository, use this priority order: + +1. Explicit user instructions, when safe. +2. `CODEX.md` as the active Codex working guide. +3. `CURRENT_SCOPE.md` for active scope and historical/sub-scope routing. +4. `DATA_CONTRACTS.md` for canonical data layout and generated artifact contracts. +5. `RUNBOOK.md` for safe commands, validation commands, and approval rules. +6. `README.md` for user-facing usage details. +7. Historical/deprecated notes in this file only when they do not conflict with active contracts. + +If these files conflict, `CODEX.md`, `CURRENT_SCOPE.md`, `DATA_CONTRACTS.md`, and `RUNBOOK.md` override historical MVP notes in this file. + +## 2. Active scope + +The active scope is the governed compact KB release. + +Canonical active pipeline: + +```text +input/raw + -> inventory + -> chunks + -> clean notes + -> source cards + -> KB build + -> judge/QA + -> publish + -> compact ChatGPT package +``` + +MVP-10 search and synthesis-layer documents are historical or sub-scope references unless the user explicitly requests those tasks. + +## 3. Canonical data layout + +The current canonical file-first layout is: + +```text +input/raw +workspace/* +publish/* +``` + +Layer rules: + +- `input/raw` is the source input layer and must remain unchanged unless raw-file maintenance is explicitly in scope. +- `workspace/*` is generated working state. +- `publish/*` is generated user-facing and governance output. +- Do not migrate this repository to `data/raw -> data/stage -> data/mart -> data/report` unless a separate migration SPEC is approved. +- Do not create a parallel `data/` structure without explicit approval. + +## 4. Network / LLM rules + +- Prefer deterministic and no-network commands by default. +- Do not run network, LLM, Ollama, or Gemini routes without explicit user approval. +- Approval must be specific to the route or command being run. +- General approval to edit documentation is not approval to run network, LLM, Ollama, Gemini, or generated-output pipelines. +- Never read or print `.env`. +- Use `.env.example` for documented environment variable names. + +## 4.1 Public migration rules + +- For GitHub/publication work, follow `docs/migration/publication_scope.md`, `docs/migration/raw_input_policy.md`, and `docs/migration/generated_artifacts_policy.md`. +- Do not publish `input/raw/` by default. +- Do not publish `workspace/`, `publish/`, `publish_assets/`, or `release_snapshots/` by default. +- Keep `.env.example` safe and placeholder-only. +- Do not add embeddings, vector DB, web UI, autonomous retrieval, or new services without explicit approval. +- Keep migration changes atomic and report changed files, checks, assumptions, risks, and acceptance status. + +## 5. Forbidden actions + +Do not: + +- modify `src/**` unless runtime code changes are explicitly in scope; +- modify `scripts/**` unless runtime command changes are explicitly in scope; +- modify `input/**` unless raw input maintenance is explicitly in scope; +- modify `workspace/**` unless generated working artifacts are explicitly in scope; +- modify `publish/**` unless generated publish/governance artifacts are explicitly in scope; +- read or print `.env`; +- run Ollama, Gemini, LLM, network, SSH, remote, deployment, or publish routes without explicit approval; +- migrate folder structure; +- add embeddings, vector DB, scheduler, web UI, or new production dependencies without explicit scope approval; +- run destructive git or filesystem commands without explicit approval; +- claim tests, checks, commands, or pipeline runs passed unless they were actually executed. + +## 6. Source discipline + +Treat as facts only: + +- user-provided context; +- repository files actually inspected; +- command outputs actually observed; +- tests/checks actually run; +- calculations performed by Python or SQL for numeric, financial, data, or analytical work; +- official documentation explicitly provided or retrieved. + +Do not invent data, files, commands, outputs, APIs, test results, package versions, or business rules. + +When uncertain, label the item: + +```text +Assumption: +Unverified: +Risk: +Blocker: +``` + +## 7. Minimal-change rule + +Make only the smallest necessary changes. + +Do not refactor, rewrite, rename, reformat, or clean up unrelated files unless required by the requested task. Every changed line must trace back to the user request, required verification, or cleanup caused by the current change. + +## 8. Validation rules + +- For documentation-only changes, verify internal consistency and run safe read/check commands from the task or `RUNBOOK.md`. +- For code changes, run the smallest relevant test or smoke check available. +- For generated-output changes, state which generated folders were intentionally changed and why. +- If a test or check cannot be run, report it as not run and explain why. +- Do not run network/LLM routes for validation unless explicitly approved. + +## 9. Historical / deprecated rules + +This section preserves older project guardrails for context. These rules are historical/deprecated and must not override `CODEX.md`, `CURRENT_SCOPE.md`, `DATA_CONTRACTS.md`, or `RUNBOOK.md`. + +### Historical MVP-1 scope + +Earlier MVP-1 notes limited implementation to: + +- Phase 0 - Bootstrap; +- Phase 1 - Inventory; +- Phase 2 - Chunking. + +Earlier MVP-1 notes also said not to implement clean notes, source cards, KB build, judge, publish, embeddings, vector DB, scheduler, or web UI unless explicitly requested. + +Current status: historical/deprecated for active release work. The active governed compact KB release now includes clean notes, source cards, KB build, judge/QA, publish, and compact ChatGPT package steps. + +### Historical MVP-1 forbidden actions + +Earlier MVP-1 notes forbade: + +- modifying raw input files; +- LLM calls; +- Ollama; +- network calls; +- SSH; +- embeddings; +- vector database; +- hallucinated synthesis. + +Current status: preserve the safety intent. For active work, use the approval matrix in `CODEX.md` and the command rules in `RUNBOOK.md`. Network, LLM, Ollama, and Gemini routes are allowed only with explicit user approval. + +### Historical analytical data layout + +Earlier notes used this analytical layout: + +```text +data/raw -> data/stage -> data/mart -> data/report +``` + +Current status: historical/deprecated for this repository. The active canonical layout is `input/raw -> workspace/* -> publish/*` as defined in `DATA_CONTRACTS.md`. + +Do not use the historical analytical layout unless a separate migration SPEC is approved. + +## 10. Handoff rule + +After each Codex execution/change run, report: + +```markdown +changed files +- ... + +commands run +- ... + +tests run +- ... + +skipped tests and reasons +- ... + +blockers +- ... + +residual risks +- ... +``` + +For read-only review tasks, use the same fields and write `none` where no files changed or no commands/tests were run. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e1f21f3 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,14 @@ +# Changelog + +## Unreleased + +### Added +- GitHub migration preparation. +- Public repository foundation files. +- Publication scope and risk documentation. + +### Changed +- Hardened repository hygiene for public migration. + +### Security +- Added publication safety rules for secrets, raw inputs, and generated artifacts. diff --git a/CODEX.md b/CODEX.md new file mode 100644 index 0000000..d64e38c --- /dev/null +++ b/CODEX.md @@ -0,0 +1,150 @@ +# CODEX.md + +Status: active +Last updated: 2026-05-21 +Purpose: active working guide for Codex in this repository. + +## 1. Active scope + +The active workflow is the governed compact KB release: + +```text +input/raw + -> inventory + -> chunks + -> clean notes + -> source cards + -> KB build + -> judge/QA + -> publish + -> compact ChatGPT package +``` + +MVP-10 search and synthesis-layer documents are historical or sub-scope references unless the user explicitly asks for those tasks. + +## 2. Source-of-truth docs + +Use this routing order before implementation: + +| Priority | File | Role | +|---|---|---| +| 1 | `CURRENT_SCOPE.md` | Active scope and historical/sub-scope routing. | +| 2 | `DATA_CONTRACTS.md` | Canonical file-first data layout and generated artifact contracts. | +| 3 | `RUNBOOK.md` | Safe setup, validation, and pipeline commands. | +| 4 | `README.md` | User-facing usage details. Treat MVP-10 sections as sub-scope when they conflict with `CURRENT_SCOPE.md`. | +| 5 | `SPEC_AGENTS_CLEANUP.md` | Known AGENTS.md conflicts and future cleanup requirements. | +| 6 | `AGENTS.md` | Historical repository guardrails. Follow strict safety rules, but route active-scope conflicts through this `CODEX.md`. | + +## 3. Canonical data layout + +The current canonical layout is: + +```text +input/raw +workspace/inventory +workspace/chunks +workspace/clean_notes +workspace/source_cards +workspace/knowledge +publish/* +``` + +The classic analytical layout `data/raw -> data/stage -> data/mart -> data/report` is not the current canonical layout for this repository. Do not migrate folders without a separate SPEC and explicit approval. + +## 4. Forbidden actions + +Do not: + +- modify `src/**` unless the user explicitly asks for runtime code changes; +- modify `scripts/**` unless the user explicitly asks for runtime command changes; +- modify `input/**` unless the user explicitly approves raw input maintenance; +- modify `workspace/**` or `publish/**` during documentation-only tasks; +- read or print `.env`; +- run network, LLM, Ollama, or Gemini routes without explicit user approval; +- migrate folder structure; +- archive, delete, or rewrite historical docs without explicit approval; +- add embeddings, vector DB, scheduler, web UI, SSH, or remote operations without explicit scope approval; +- claim tests, checks, or pipeline runs passed unless actually executed. + +## 5. Network / LLM approval matrix + +| Action | Default | Approval required | Notes | +|---|---|---|---| +| Read docs and source files | Allowed | No | Do not read `.env`. | +| Run `rg`, `sed`, `find`, `git status` | Allowed | No | Read/check only. | +| Run targeted pytest | Allowed | No | Prefer targeted tests from `RUNBOOK.md`. | +| Run deterministic no-network managed pipeline | Not automatic | Yes when it changes generated outputs | Must use `--use-ollama off --use-gemini off`. | +| Run Ollama route | Forbidden by default | Yes | Local model call; may depend on local service. | +| Run Gemini route | Forbidden by default | Yes | Network/API route and `.env`-dependent. | +| Print or inspect `.env` | Forbidden | Do not request for normal work | Use `.env.example` for documented keys. | +| SSH, deployment, publish, remote git push | Forbidden by default | Yes | Treat as high risk. | + +Approval must be specific to the action. General permission to edit docs is not approval to run network, LLM, Ollama, Gemini, or generated-output pipelines. + +## 6. Safe validation commands + +Documentation contract checks: + +```bash +rg -n "governed compact KB release|CURRENT_SCOPE|DATA_CONTRACTS|RUNBOOK" CODEX.md AGENTS.md README.md +rg -n "Ollama|Gemini|network|approval|.env" CODEX.md AGENTS.md README.md +git status --short +``` + +Optional targeted test: + +```bash +python3 -m pytest tests/test_search_cli_mvp_10.py -q +``` + +Additional safe test commands are listed in `RUNBOOK.md`. + +## 7. Generated artifact policy + +- `input/raw` is source material and must remain unchanged unless explicitly approved. +- `workspace/*` is generated working state. +- `publish/*` is generated user-facing and governance output. +- Documentation-only tasks must not modify `workspace/*` or `publish/*`. +- Pipeline tasks may modify generated artifacts only when the user explicitly asks for a pipeline run or generated-output refresh. +- Do not upload or treat raw transcripts, chunks, clean notes, or source cards as the compact ChatGPT package unless explicitly requested. + +For public GitHub migration, use these additional policies: + +- `docs/migration/publication_scope.md` +- `docs/migration/raw_input_policy.md` +- `docs/migration/generated_artifacts_policy.md` + +Raw transcripts and generated outputs are excluded from publication by default. Publish only reviewed, explicit artifacts. + +## 8. Stale-doc handling + +When docs conflict: + +1. Follow explicit user instructions if safe. +2. Follow `CURRENT_SCOPE.md`, `DATA_CONTRACTS.md`, and `RUNBOOK.md` for the active governed compact KB release. +3. Treat MVP-1, MVP-10, and synthesis-layer docs as historical/sub-scope unless the task explicitly names them. +4. Preserve stricter safety rules from `AGENTS.md` when they do not conflict with the active scope. +5. If a conflict affects implementation or validation, state it before changing files. + +## 9. Handoff format after each Codex run + +Use this handoff for execution/change tasks: + +```markdown +changed files +- ... + +commands run +- ... + +test results +- ... + +unresolved conflicts +- ... + +whether AGENTS.md still needs manual review +- yes/no, with one short reason +``` + +For read-only review tasks, keep the same structure and write `none` where no files changed. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..d8ac22b --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,58 @@ +# Contributing + +Проект готовит управляемую компактную базу знаний из заметок и транскриптов с проверяемыми промежуточными шагами. + +## Локальная настройка + +```bash +python3 -m venv .venv +source .venv/bin/activate +python3 -m pip install -r requirements.txt +``` + +Для маршрутов Gemini/Ollama используйте `.env.example` как шаблон. Файл `.env` локальный, игнорируется git и не должен попадать в коммиты. + +## Ветки + +Используйте короткие ветки с понятным назначением: + +```text +chore/ +docs/ +fix/ +``` + +## Проверки + +Минимальная проверка перед PR: + +```bash +python3 -m pytest -q +``` + +Если полный набор тестов зависит от локальных приватных данных или окружения, укажите это в PR и запустите самый узкий воспроизводимый набор. + +## PR checklist + +- Изменения атомарные и относятся к заявленной задаче. +- Нет `.env`, ключей, токенов, паролей или приватных путей. +- Нет raw-транскриптов из `input/raw/`. +- Нет сгенерированных артефактов из `workspace/`, `publish/`, `publish_assets/`, `release_snapshots/`, если они не утверждены отдельно. +- `.env.example` содержит только безопасные placeholder-значения. +- Запущены релевантные тесты или явно указан блокер. + +## Секреты и приватные данные + +Никогда не печатайте и не коммитьте `.env`. Для документации переменных окружения используйте только `.env.example` и placeholder-значения. + +## Raw data policy + +`input/raw/` приватен по умолчанию. Публиковать можно только заранее очищенные примеры без персональных данных, приватных серверных деталей, ключей, токенов, финансовых или операционных деталей и спорного исходного материала. + +## Generated artifacts policy + +`workspace/`, `publish/`, `publish_assets/` и `release_snapshots/` являются рабочими или выпускными артефактами и не публикуются по умолчанию. Финальные артефакты можно добавлять позже только после отдельного review. + +## Business logic + +Не меняйте бизнес-логику, схемы, форматы выходных файлов, определения метрик или pipeline-контракты без отдельного review и явного scope. diff --git a/CURRENT_SCOPE.md b/CURRENT_SCOPE.md new file mode 100644 index 0000000..6fca892 --- /dev/null +++ b/CURRENT_SCOPE.md @@ -0,0 +1,59 @@ +# CURRENT_SCOPE.md + +Status: active +Last updated: 2026-05-21 + +## Active scope + +The active project scope is the governed compact KB release. + +Canonical active workflow: + +```text +input/raw + -> inventory + -> chunks + -> clean notes + -> source cards + -> KB build + -> judge/QA + -> publish + -> compact ChatGPT package +``` + +## Source of truth + +Use these files as the current source of truth for normal Codex work: + +| File | Status | Purpose | +|---|---|---| +| `CURRENT_SCOPE.md` | active | Current operating scope and doc routing. | +| `DATA_CONTRACTS.md` | active | Current file-first data layout and generated artifact contracts. | +| `RUNBOOK.md` | active | Safe setup, validation, and pipeline commands. | +| `README.md` | active with historical sections | User-facing usage guide for search, pipeline, publish, and compact package workflows. | +| `AGENTS.md` | active but needs cleanup | Repository-level guardrails; known conflicts are documented in `SPEC_AGENTS_CLEANUP.md`. | + +## Historical or sub-scope documents + +These files are still useful, but they must not override the active governed compact KB release scope unless a user explicitly requests that sub-scope. + +| File | Status | Notes | +|---|---|---| +| `SPEC.md` | historical/sub-scope | Describes MVP-10 Lightweight Search CLI, not the full active governed release. | +| `SCOPE_LOCK.md` | historical/sub-scope | Scope lock for MVP-10 search work. | +| `ACCEPTANCE_CRITERIA.md` | historical/sub-scope | MVP-10 search acceptance criteria. | +| `MODEL_ROUTING.md` | historical/sub-scope | MVP-10 search rule: no model routing. | +| `SKILLS_SEQUENCE.md` | historical/sub-scope | MVP-10 search workflow sequence. | +| `SPEC_synthesis_layer.md` | sub-scope | Synthesis-layer task spec for compact KB outputs. | +| `plan.md` | sub-scope | Synthesis-layer implementation plan. | +| `tasks.md` | sub-scope | Synthesis-layer completed task checklist. | +| `scope-lock.md` | sub-scope | Synthesis-layer scope lock. | +| `implementation-guard.md` | sub-scope | Synthesis-layer implementation guard. | + +## Operating rules + +- Treat `.txt` files under `input/raw` as source material. +- Do not modify `input/raw`, `workspace`, or `publish` unless a task explicitly permits generated-output changes. +- Do not read or print `.env`. +- Do not run network, LLM, Ollama, or Gemini routes without explicit user approval. +- For documentation-only work, do not run runtime pipelines. diff --git a/DATA_CONTRACTS.md b/DATA_CONTRACTS.md new file mode 100644 index 0000000..570522c --- /dev/null +++ b/DATA_CONTRACTS.md @@ -0,0 +1,55 @@ +# DATA_CONTRACTS.md + +Status: active +Last updated: 2026-05-21 + +## Current canonical layout + +This repository currently uses a file-first Knowledge Base layout, not the classic analytical `data/raw -> data/stage -> data/mart -> data/report` layout. + +Current canonical flow: + +```text +input/raw + -> workspace/inventory + -> workspace/chunks + -> workspace/clean_notes + -> workspace/source_cards + -> workspace/knowledge + -> publish/* +``` + +## Layer contracts + +| Path | Layer | Contract | +|---|---|---| +| `input/raw` | raw input | Source transcript files. Treat as read-only source material. Do not overwrite, move, normalize, or delete through pipeline runs. | +| `workspace/inventory` | inventory | Generated file and source indexes derived from `input/raw`. Rebuildable from raw inputs. | +| `workspace/chunks` | chunk stage | Generated transcript chunks and chunk manifests. Rebuildable from inventory and raw inputs. | +| `workspace/clean_notes` | clean-note stage | Generated clean notes derived from chunks and a configured provider. Existing notes may be reused by resume workflows. | +| `workspace/source_cards` | source-card stage | Generated source cards with evidence and QA metadata. These are the main traceability artifacts for KB build. | +| `workspace/knowledge` | KB build and QA workspace | Generated topics, concepts, indexes, judge reports, review queues, conflicts, and unsupported-claims reports. | +| `publish/chatgpt_project` | publish output | User-facing ChatGPT Project package. | +| `publish/chatgpt_project_compact` | publish output | Compact ChatGPT Project package and governed release files. | +| `publish/markdown_kb` | publish output | Local markdown KB and lexical search target. | +| `publish/obsidian` | publish output | Obsidian-oriented markdown package. | +| `publish/*.json` and `publish/*.md` | governance output | Runtime governance, QA, acceptance, promotion, and release reports. | + +## Explicit non-canonical layout + +The following analytical layout is not the current canonical layout for this repository: + +```text +data/raw -> data/stage -> data/mart -> data/report +``` + +Do not migrate folders or create a parallel `data/` structure without a separate SPEC and explicit approval. + +## Safety rules + +- `input/raw` is the source input layer and must remain unchanged unless the user explicitly asks for raw-file maintenance. +- `workspace/*` is generated working state. +- `publish/*` is generated user-facing and governance output. +- Raw transcripts, chunks, clean notes, and source cards should not be uploaded as the compact ChatGPT package unless a task explicitly says so. +- Network, LLM, Ollama, and Gemini routes require explicit user approval before execution. +- Never read or print `.env`. diff --git a/EXPORT_MANIFEST.md b/EXPORT_MANIFEST.md new file mode 100644 index 0000000..1996650 --- /dev/null +++ b/EXPORT_MANIFEST.md @@ -0,0 +1,44 @@ +# Export Manifest + +## Purpose + +This directory is a clean public export prepared for GitHub migration. + +## Destination + +https://github.com/sergstack/Build-your-knowledge-base + +## Included + +- source code; +- scripts; +- tests; +- prompts; +- public documentation; +- GitHub templates; +- safe config examples. + +## Excluded + +- `.git/`; +- `.env`; +- raw transcripts; +- private inputs; +- generated workspace files; +- generated publication outputs; +- release snapshots; +- caches; +- logs; +- internal migration-risk reports. + +## Why clean export is required + +The original local repository is not pushed directly because it may contain private/raw/generated material in the working tree or history. + +The safe migration route is: + +`clean export -> clean GitHub clone -> branch -> PR`. + +## Validation + +See `PUBLICATION_AUDIT.md`. diff --git a/PHASE2_GITHUB_TRANSFER_INSTRUCTIONS.md b/PHASE2_GITHUB_TRANSFER_INSTRUCTIONS.md new file mode 100644 index 0000000..5978646 --- /dev/null +++ b/PHASE2_GITHUB_TRANSFER_INSTRUCTIONS.md @@ -0,0 +1,42 @@ +# Phase 2 GitHub Transfer Instructions + +## Strategy + +Use a clean clone of the destination GitHub repository and copy the public export into a new branch. + +Do not push the original source repository history. + +## Destination + +https://github.com/sergstack/Build-your-knowledge-base + +## Branch + +`chore/github-migration-foundation` + +## Required checks before push + +```bash +git status --short +git diff --stat +git diff --check +python3 -m pytest tests -q +``` + +## Forbidden + +- no force push; +- no direct push to `main`; +- no source repository history push; +- no raw transcripts; +- no `.env`; +- no generated artifacts; +- no merge without user approval. + +## Acceptance + +- tests pass; +- PR file list reviewed; +- no private data included; +- CI passes; +- user approves merge. diff --git a/PUBLICATION_AUDIT.md b/PUBLICATION_AUDIT.md new file mode 100644 index 0000000..7941b19 --- /dev/null +++ b/PUBLICATION_AUDIT.md @@ -0,0 +1,65 @@ +# Publication Audit + +## Summary + +This export is prepared as a public-safe GitHub migration package. + +## Public safety status + +Status: pending final PR review. + +## Included file groups + +- source code; +- scripts; +- tests; +- prompts; +- public documentation; +- GitHub templates; +- safe configuration examples. + +## Excluded file groups + +- secrets and local environment files; +- raw/private inputs; +- generated runtime workspace; +- generated publication outputs; +- release snapshots; +- caches and logs; +- internal migration-risk details. + +## Secret scan result + +A redacted secret scan must be run before PR creation. + +Any actual credential is a blocker. + +## Raw input policy + +Raw/private inputs are excluded by default. Only sanitized samples may be published. + +## Generated artifact policy + +Generated outputs are excluded by default. Approved final artifacts may be attached to releases later after review. + +## Test result + +`python3 -m pytest tests -q` passed in the clean export with `66 passed`. + +The public export excludes generated publication artifacts by design. Legacy tests that require pre-existing generated `publish/` files are skipped by export-local collection rules when those artifacts are absent. + +## Remaining user decisions + +- whether to publish sanitized sample data later; +- whether to publish selected release artifacts later; +- whether to merge the PR after CI passes. + +## Phase 2 recommendation + +Use branch + PR. Do not force-push and do not push the original local repository history. + +## Final local secret scan + +Completed before PR preparation. + +Result: no live credential value was confirmed. Hits were limited to safe placeholders, code variables, test fixtures, and documentation warnings. diff --git a/PUBLICATION_CHECKLIST.md b/PUBLICATION_CHECKLIST.md new file mode 100644 index 0000000..af16632 --- /dev/null +++ b/PUBLICATION_CHECKLIST.md @@ -0,0 +1,15 @@ +# Publication Checklist + +Status: Phase 1 migration preparation + +- [ ] Working tree contains only approved migration-prep changes. +- [ ] No `.env` file is staged or committed. +- [ ] `.env.example` contains only placeholders. +- [ ] `input/raw/` is excluded from publication unless sanitized samples are approved. +- [ ] `workspace/`, `publish/`, `publish_assets/`, and `release_snapshots/` are excluded unless selected artifacts are approved. +- [ ] Redacted secret scan has been run after final edits. +- [ ] Tests have passed or failures are documented. +- [ ] GitHub remote has been inspected before any push. +- [ ] Destination README has been integrated by PR, not overwritten blindly. +- [ ] PR file list has been reviewed manually. +- [ ] User has approved Phase 2 push/PR. diff --git a/README.md b/README.md index 0ebe4b2..e8bfd00 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,354 @@ -# Build-your-knowledge-base -Подготовка Компактной Базы знаний ИИ +# Build Your Knowledge Base + +## Purpose + +A governed Python pipeline for turning notes/transcripts into a structured, reviewable knowledge base package. + +This repository is being prepared for public GitHub migration. Raw inputs are private by default and are not included in the public repository. Generated outputs are excluded by default. + +## Current Scope + +```text +Governed compact KB release: +input/raw -> inventory -> chunks -> clean notes -> source cards -> KB build -> judge/QA -> publish -> compact ChatGPT package +``` + +Use `CURRENT_SCOPE.md`, `DATA_CONTRACTS.md`, `RUNBOOK.md`, and `CODEX.md` as the active operating contracts. MVP-10 search files remain available as sub-scope documentation when search-specific work is requested. + +## Repository Structure + +| Path | Purpose | Publication default | +|---|---|---| +| `src/notes_to_kb/` | Python package code | publish | +| `scripts/` | CLI and pipeline entry points | publish | +| `tests/` | pytest coverage | publish | +| `prompts/` | prompt templates | publish after review | +| `docs/migration/` | migration policies and readiness evidence | publish | +| `.github/` | GitHub PR, issue, and CI foundation | publish | +| `input/raw/` | private raw transcripts | do not publish | +| `workspace/` | generated working state | do not publish | +| `publish/` | generated release outputs | do not publish by default | +| `publish_assets/` | generated/publication support | do not publish by default | +| `release_snapshots/` | release evidence snapshots | do not publish by default | + +See `docs/migration/publication_scope.md`, `docs/migration/raw_input_policy.md`, and `docs/migration/generated_artifacts_policy.md` before preparing a public transfer. + +## Files + +| File | Purpose | +|---|---| +| `CURRENT_SCOPE.md` | Active scope and source-of-truth routing | +| `DATA_CONTRACTS.md` | Current file-first data layout | +| `RUNBOOK.md` | Safe setup, validation, and pipeline commands | +| `CODEX.md` | Active Codex working guide | +| `SPEC.md` | Main MVP-10 specification | +| `PLAN.md` | Work packages and execution order | +| `TASKS.md` | Implementation checklist | +| `SCOPE_LOCK.md` | Allowed and forbidden changes | +| `ACCEPTANCE_CRITERIA.md` | Completion gates | +| `MODEL_ROUTING.md` | Confirms no model routing for MVP-10 | +| `SKILLS_SEQUENCE.md` | Minimal workflow sequence | + +## Search CLI + +Run lexical search over the default published markdown KB: + +```bash +python3 scripts/search_kb.py Knowledge --limit 3 +``` + +Default search file: + +```text +publish/markdown_kb/full_kb.md +``` + +JSON output: + +```bash +python3 scripts/search_kb.py Knowledge --limit 1 --json +``` + +Search a specific published file: + +```bash +python3 scripts/search_kb.py "source card" --path publish/chatgpt_project/TRACEABILITY_GUIDE.md --limit 5 +``` + +The search is deterministic lexical matching only. It does not use LLMs, Ollama, embeddings, a vector DB, or network calls. + +Options: + +| Option | Purpose | +|---|---| +| positional `query` | Required search text. | +| `--path` | Optional markdown or CSV file to search. Relative paths are resolved from the project root. | +| `--limit` | Maximum result count. Default: `10`. | +| `--json` | Emit deterministic JSON with `query`, `path`, and `results`. | + +Failure behavior: + +- empty query returns a clear error; +- missing search file returns a clear error; +- no matches returns `result_count=0` or an empty JSON `results` list. + +## Hard Stop + +Do not implement: + +- embeddings; +- vector DB; +- semantic RAG; +- web UI; +- scheduler; +- Mode B; +- Mode C; +- LLM calls; +- network calls. + +This hard stop applies to the lightweight search CLI. The transcript processing pipeline below can use Ollama and Gemini when explicitly run. + +## Transcript Processing Pipeline + +The current supported full route processes `.txt` transcripts into clean notes, source cards, deterministic KB sections, judge reports, and publish outputs. + +Route: + +| Stage | Engine | +|---|---| +| Inventory | deterministic Python | +| Chunking | deterministic Python | +| Raw transcript -> clean note | Ollama | +| Clean note + chunks -> source card | Gemini 2.5 Flash-Lite | +| Source cards -> KB sections | deterministic Python | +| Final judge | deterministic Python | +| Publish | deterministic Python | +| Core tests | pytest | + +This route does not use embeddings, vector DB, semantic search, web UI, scheduler, Mode B, or Mode C. + +### Setup + +Install local test/runtime dependencies: + +```bash +python3 -m venv .venv +source .venv/bin/activate +python3 -m pip install -r requirements.txt +``` + +Create `.env` from `.env.example` and fill the Gemini key: + +```bash +cp .env.example .env +``` + +Required for the Gemini route: + +```text +GEMINI_API_KEY=... +``` + +Common optional settings: + +```text +GEMINI_SOURCE_CARD_MODEL=gemini-2.5-flash-lite +GEMINI_TIMEOUT=600 +GEMINI_MAX_OUTPUT_TOKENS=4096 +GEMINI_THINKING_BUDGET=0 +GEMINI_MAX_RETRIES=3 +OLLAMA_BASE_URL=http://127.0.0.1:11434 +CLEAN_MODEL=devstral-small-2:24b-instruct-2512-q4_K_M +``` + +Ollama must be running locally before launch. The command wrapper checks `OLLAMA_BASE_URL/api/tags` before starting. + +Do not commit `.env`. Keep `.env.example` limited to placeholder values. + +### Run + +From the project root: + +```bash +./run_update_compact_ollama_gemini.command +``` + +You can also double-click: + +```text +run_update_compact_ollama_gemini.command +``` + +The wrapper resumes safely: + +- existing clean notes are not regenerated; +- existing source cards are not overwritten; +- missing source cards continue from where the previous run stopped; +- transient Gemini timeouts and 429/5xx responses are retried; +- publish is allowed when judge readiness is `needs_review`; +- publish remains blocked when judge readiness is `blocked`. + +Source card QA defaults to relaxed mode. `relaxed` sends only empty, broken, unusable, or clearly irrelevant cards to review; formal issues such as section order, extra sections, removed `` blocks, or incomplete chunk evidence are recorded as warnings. Use `--qa-strictness standard` or `--qa-strictness strict` with `scripts/run_source_card.py` when stricter gating is needed. + +### Output Files + +Main generated working files: + +```text +workspace/inventory/files_index.csv +workspace/inventory/sources_index.csv +workspace/chunks/ +workspace/clean_notes/ +workspace/source_cards/ +workspace/knowledge/topics/ +workspace/knowledge/concepts/ +workspace/knowledge/indexes/ +workspace/knowledge/reports/judge_report.md +``` + +Ready-to-use publish files: + +```text +publish/chatgpt_project/AI_KB_Context_File_v1.0.md +publish/chatgpt_project/INDEX.md +publish/chatgpt_project/CONCEPT_MAP.md +publish/chatgpt_project/WORKFLOW_MAP.md +publish/chatgpt_project/TRACEABILITY_GUIDE.md +publish/chatgpt_project/KB_USAGE_GUIDE.md +publish/chatgpt_project/SMOKE_QUESTIONS.md + +publish/markdown_kb/full_kb.md +publish/markdown_kb/sources_index.csv +publish/markdown_kb/concepts_index.csv + +publish/obsidian/ +``` + +### How To Use The Publish + +For ChatGPT Projects: + +1. Open `publish/chatgpt_project/`. +2. Upload `AI_KB_Context_File_v1.0.md` as the main context file. +3. Upload the helper files from the same folder when you want navigation, traceability, workflow, and smoke-question support. +4. Use `SMOKE_QUESTIONS.md` to check whether the project answers from the KB rather than inventing. + +For a smaller ChatGPT Project upload package, build the compact seven-file package: + +```bash +python3 scripts/build_chatgpt_compact_kb.py +``` + +Output: + +```text +publish/chatgpt_project_compact/ +``` + +Upload all files from `publish/chatgpt_project_compact/` into the same ChatGPT Project. Do not upload raw transcripts, clean notes, source cards, chunks, temp files, logs, embeddings, or vector database files. + +To add the deterministic synthesis layer with canonical concepts, frameworks, patterns, anti-patterns, and Sergey-focused use cases: + +```bash +python3 scripts/build_synthesis_layer.py +``` + +This adds: + +```text +publish/chatgpt_project_compact/KB__05_CANONICAL_CONCEPTS.md +publish/chatgpt_project_compact/KB__06_OPERATIONAL_FRAMEWORKS.md +publish/chatgpt_project_compact/KB__07_PATTERNS_AND_FAILURES.md +publish/chatgpt_project_compact/KB__08_USE_CASES_FOR_SERGEY.md +publish/chatgpt_project_compact/SYNTHESIS_MANIFEST.md +``` + +The synthesis layer is deterministic by default. Weak evidence is marked explicitly instead of being promoted to finished knowledge. + +For local markdown reading or search: + +```bash +python3 scripts/search_kb.py "variance analysis" --path publish/markdown_kb/full_kb.md --limit 5 +``` + +For Obsidian: + +1. Copy or open `publish/obsidian/` as a vault or inside an existing vault. +2. Use `publish/obsidian/INDEX.md` as the entry point. +3. Topic files are under `topics/`, concepts under `concepts/`, and per-source cards under `sources/`. + +Judge output: + +```text +workspace/knowledge/reports/judge_report.md +``` + +`needs_review` means the publish was created, but some source cards are marked for human review. `blocked` means publish should not be used without fixing the reported issues or explicitly overriding the publish command. + +### Core Validation + +The Gemini wrapper runs the core pipeline tests: + +```bash +python3 -m pytest \ + tests/test_inventory.py \ + tests/test_chunking.py \ + tests/test_clean_note.py \ + tests/test_source_card.py \ + tests/test_kb_build.py \ + tests/test_judge.py \ + tests/test_publish.py \ + tests/test_ollama_client.py +``` + +The broader `python3 -m pytest tests` suite also includes legacy/manual QA release artifact checks. Those are not part of the current `.txt` processing route. + +For public GitHub migration checks, run: + +```bash +python3 -m pytest tests -q +git diff --check +``` + +## Data and Privacy Policy + +`input/raw/` is private by default. Raw transcripts must not be published unless they are explicitly reviewed and sanitized. Sanitized samples must remove credentials, personal data, private server details, API keys, tokens, financial/private operational details, and copyrighted/private source material. + +Do not read, print, commit, or upload `.env`. + +## Generated Artifacts Policy + +`workspace/`, `publish/`, `publish_assets/`, and `release_snapshots/` are generated or release-output folders. They are ignored by default for public repository migration. Selected release artifacts can be published later only after explicit review, preferably through GitHub Releases rather than source commits. + +## GitHub Migration Status + +Phase 1 migration preparation is local-only. Do not push, force-push, add remotes, or overwrite the destination repository in this phase. The recommended transfer path is documented in `docs/migration/github_transfer_plan.md`. + +Current readiness is partial until dirty runtime/test changes, raw transcript publication decisions, and selected generated artifact decisions are resolved. + +## Limitations + +- The default search CLI is deterministic lexical search, not semantic RAG. +- Embeddings, vector DB, web UI, scheduler, and autonomous retrieval are out of scope unless explicitly approved. +- LLM/Ollama/Gemini routes require explicit approval and local/API configuration. +- Generated outputs may reflect private raw inputs and should not be treated as public-safe without review. + +## Ollama-Only Pipeline + +Clean note and source card generation can use a local Ollama server instead of the deterministic mock provider: + +```bash +python3 scripts/run_clean_note.py --all --provider ollama --model --overwrite +python3 scripts/run_source_card.py --all --provider ollama --model --overwrite +python3 scripts/run_kb_build.py --all --overwrite +python3 scripts/run_judge.py --all +python3 scripts/run_publish.py --mode all +``` + +The wrapper runs the current supported pipeline in the same order: + +```bash +python3 scripts/run_ollama_pipeline.py --model +``` + +By default, Ollama requests use `http://127.0.0.1:11434`. Override it with `--ollama-base-url` or `OLLAMA_BASE_URL`. diff --git a/RUNBOOK.md b/RUNBOOK.md new file mode 100644 index 0000000..8cf9844 --- /dev/null +++ b/RUNBOOK.md @@ -0,0 +1,109 @@ +# RUNBOOK.md + +Status: active +Last updated: 2026-05-21 + +## Safety rules + +- Never read or print `.env`. +- Do not run network, LLM, Ollama, or Gemini routes without explicit user approval. +- Do not modify `input/raw`, `workspace`, or `publish` during documentation-only tasks. +- Prefer targeted pytest commands before broad checks. + +## Setup + +Create a local virtual environment and install declared Python dependencies: + +```bash +python3 -m venv .venv +source .venv/bin/activate +python3 -m pip install -r requirements.txt +``` + +For Gemini-assisted routes, create `.env` from `.env.example` and fill values manually. Do not print the file contents. + +```bash +cp .env.example .env +``` + +## Targeted tests + +Search CLI: + +```bash +python3 -m pytest tests/test_search_cli_mvp_10.py -q +``` + +Core deterministic pipeline modules: + +```bash +python3 -m pytest \ + tests/test_inventory.py \ + tests/test_chunking.py \ + tests/test_kb_build.py \ + tests/test_judge.py \ + tests/test_publish.py \ + -q +``` + +Clean note and source card behavior: + +```bash +python3 -m pytest \ + tests/test_clean_note.py \ + tests/test_source_card.py \ + -q +``` + +Full suite: + +```bash +python3 -m pytest tests -q +``` + +## Deterministic no-network pipeline + +Use this route when generated outputs are allowed to change and no network, Ollama, or Gemini calls are approved: + +```bash +python3 scripts/run_managed_knowledge_factory.py \ + --mode full \ + --use-ollama off \ + --use-gemini off +``` + +For a faster validation run that skips the final pytest call inside the managed pipeline: + +```bash +python3 scripts/run_managed_knowledge_factory.py \ + --mode full \ + --use-ollama off \ + --use-gemini off \ + --skip-tests +``` + +## LLM/Ollama/Gemini route + +This route requires explicit user approval because it can use local Ollama, Gemini, network access, and `.env` configuration: + +```bash +./run_update_compact_ollama_gemini.command +``` + +Ollama-only assisted route, also requiring explicit user approval: + +```bash +./run_update_compact_ollama.command +``` + +Approval must be specific to running the route. General permission to edit documentation is not approval to run network, LLM, Ollama, or Gemini commands. + +## Documentation-only validation + +Safe read/check commands for documentation contract changes: + +```bash +rg -n "active|historical|source of truth" CURRENT_SCOPE.md +rg -n "input/raw|workspace|publish" DATA_CONTRACTS.md +rg -n "pytest|network|Ollama|Gemini|approval" RUNBOOK.md +``` diff --git a/SPEC.md b/SPEC.md new file mode 100644 index 0000000..841825c --- /dev/null +++ b/SPEC.md @@ -0,0 +1,73 @@ +# SPEC + +## Goal + +Implement `MVP-10 Lightweight Search CLI`: a deterministic local command-line search tool for the already published knowledge base. + +The CLI should help a user search the existing published package without adding embeddings, vector database, web UI, scheduler, Mode B, Mode C, network calls, or LLM calls. + +## Current state + +- MVP-9.4 acceptance status is `ready_for_mvp_10`. +- The accepted next scope is `MVP-10 Lightweight Search CLI`. +- Published searchable artifacts already exist under: + - `publish/markdown_kb/full_kb.md`; + - `publish/markdown_kb/sources_index.csv`; + - `publish/markdown_kb/concepts_index.csv`; + - `publish/chatgpt_project/*.md`. +- Existing project CLIs live under `scripts/`. +- Existing reusable code lives under `src/notes_to_kb/`. +- Existing tests live under `tests/`. + +## Requirements + +- Add a local CLI script for lightweight KB search. +- Search must be lexical and deterministic. +- Search inputs must be existing published markdown/CSV files only. +- Default search target should be `publish/markdown_kb/full_kb.md`. +- CLI should support: + - query text; + - optional search path; + - configurable result limit; + - readable text output; + - JSON output for deterministic tests or downstream tooling. +- Results should include enough context to inspect the match. +- Empty or missing queries must fail with a clear error. +- Missing search files must fail with a clear error. +- Add deterministic tests for search behavior and CLI output. +- Record MVP-10 acceptance after tests pass. + +## Constraints + +- Minimal necessary changes only. +- Do not modify raw input files. +- Do not modify publish source data except for MVP-10 acceptance output. +- Do not use LLM calls. +- Do not use Ollama. +- Do not use network calls. +- Do not create embeddings. +- Do not add a vector database. +- Do not add web UI, scheduler, Mode B, Mode C, API server, or background jobs. +- Do not change model routing. +- Do not add external dependencies. +- Do not make search results semantic; lexical matching only. + +## Acceptance criteria + +- Search module exists and is covered by tests. +- CLI script exists and can run from the project root. +- Text output returns matching snippets for a known query. +- JSON output is valid JSON and includes query, searched path, and results. +- Result limit is respected. +- Missing query and missing file failures are tested. +- Full pytest passes. +- No `.txt` files are added to `publish/`. +- No forbidden artifacts are added to `publish/`. +- `publish/ACCEPTANCE_CHECK_MVP_10.md` records status `mvp_10_complete`. + +## Risks + +- Lexical search may miss conceptually related content when words do not match. +- Large published files may require result limits to keep output readable. +- Search snippets may expose broad context but should not rewrite or synthesize source content. +- Users may expect semantic search; this scope explicitly does not provide it. diff --git a/SPEC_AGENTS_CLEANUP.md b/SPEC_AGENTS_CLEANUP.md new file mode 100644 index 0000000..38dcd0c --- /dev/null +++ b/SPEC_AGENTS_CLEANUP.md @@ -0,0 +1,71 @@ +# SPEC_AGENTS_CLEANUP.md + +Status: draft +Last updated: 2026-05-21 + +## Goal + +Prepare a minimal cleanup of repository-level AI/Codex instructions without changing runtime code or generated artifacts. + +This SPEC does not approve editing `AGENTS.md`; it documents the required cleanup scope for a future approved task. + +## Current state + +- `AGENTS.md` says the current implementation scope is MVP-1 Phase 0-2: bootstrap, inventory, and chunking. +- `AGENTS.md` forbids LLM calls, Ollama, network calls, clean notes, source cards, KB build, judge, and publish unless explicitly requested. +- `README.md` and current scripts document and support the governed compact KB release route: + +```text +input/raw -> inventory -> chunks -> clean notes -> source cards -> KB build -> judge/QA -> publish -> compact ChatGPT package +``` + +- README and code support Ollama/Gemini-assisted routes. +- Current operating contracts are now documented in `CURRENT_SCOPE.md`, `DATA_CONTRACTS.md`, and `RUNBOOK.md`. + +## Conflicts to resolve + +| Conflict | Evidence | Risk | +|---|---|---| +| `AGENTS.md` active scope is MVP-1, while project active scope is governed compact KB release. | `AGENTS.md`; `CURRENT_SCOPE.md`; README pipeline docs. | Codex may refuse valid release tasks or follow obsolete scope. | +| `AGENTS.md` forbids LLM/Ollama/network broadly for MVP-1. | `AGENTS.md` MVP-1 forbidden actions. | Codex may treat approved Ollama/Gemini routes as always forbidden. | +| README/code support Ollama/Gemini routes. | README run sections; wrapper scripts; `src/notes_to_kb/llm_client.py`. | Safety rules are unclear without an approval matrix. | +| `AGENTS.md` mandates `data/raw -> data/stage -> data/mart -> data/report`. | `AGENTS.md` mandatory data folder rule. | Codex may create incorrect folders instead of using `input/raw -> workspace -> publish`. | +| Historical MVP docs and active release docs coexist in root. | `SPEC.md`, `SCOPE_LOCK.md`, `plan.md`, `tasks.md`, `scope-lock.md`. | Codex may select the wrong source of truth. | + +## Requirements for future cleanup + +- Preserve strict rules for source discipline, reproducibility, and no invented results. +- Split active governed compact KB release rules from historical MVP-1 and MVP-10 rules. +- State that `input/raw -> workspace -> publish` is the current canonical layout for this repository. +- State that `data/raw -> data/stage -> data/mart -> data/report` is not the current canonical layout here. +- Require explicit user approval before running network, LLM, Ollama, Gemini, or `.env`-dependent routes. +- Keep the rule: never read or print `.env`. +- Do not weaken raw input safety: `input/raw` remains read-only unless explicitly approved. +- Keep forbidden actions for embeddings, vector DB, web UI, scheduler, SSH, destructive git, and raw input modification unless a future SPEC explicitly changes them. + +## Non-goals + +- Do not modify runtime code. +- Do not migrate folders. +- Do not add dependencies. +- Do not run pipeline commands. +- Do not rewrite historical specs. +- Do not change generated outputs in `workspace` or `publish`. + +## Acceptance criteria + +- Future `AGENTS.md` cleanup clearly identifies active and historical scope. +- Codex can determine the source of truth before execution. +- Network/LLM/Ollama/Gemini usage is allowed only with explicit user approval. +- Current file-first data layout is documented as canonical for this repository. +- Historical MVP rules remain available as historical context without overriding active scope. + +## Validation + +Use documentation-only checks: + +```bash +rg -n "active|historical|source of truth" CURRENT_SCOPE.md +rg -n "input/raw|workspace|publish" DATA_CONTRACTS.md +rg -n "pytest|network|Ollama|Gemini|approval" RUNBOOK.md +``` diff --git a/SPEC_chatgpt_compact_kb.md b/SPEC_chatgpt_compact_kb.md new file mode 100644 index 0000000..a825748 --- /dev/null +++ b/SPEC_chatgpt_compact_kb.md @@ -0,0 +1,117 @@ +# SPEC + +## Goal + +Create a compact markdown package for uploading into ChatGPT Project from existing published Knowledge Base outputs. + +The package should reduce file count, preserve navigation and traceability, and avoid raw or noisy source material. + +## Current state + +- Project root is `/Users/sst/Documents/Python Progect/[Транскрибация] Knoweledge Base/`. +- Existing `SPEC.md` describes a different MVP-10 lightweight search CLI task and must not be overwritten by this spec. +- Required source folders exist: + - `publish/chatgpt_project/` + - `publish/markdown_kb/` +- Observed files under `publish/chatgpt_project/`: + - `AI_KB_Context_File_v1.0.md` + - `CONCEPT_MAP.md` + - `INDEX.md` + - `KB_USAGE_GUIDE.md` + - `SMOKE_QUESTIONS.md` + - `TRACEABILITY_GUIDE.md` + - `WORKFLOW_MAP.md` +- Observed files under `publish/markdown_kb/`: + - `concepts_index.csv` + - `full_kb.md` + - `sources_index.csv` +- Existing scripts live under `scripts/`. + +## Requirements + +- Add `scripts/build_chatgpt_compact_kb.py`. +- The script must run from the project root with: + - `python3 scripts/build_chatgpt_compact_kb.py` +- Optional CLI flags: + - `--max-file-mb 1` + - `--output publish/chatgpt_project_compact` +- Use only Python standard library. +- Create `publish/chatgpt_project_compact/` if missing. +- Overwrite previous compact output files safely. +- Preserve UTF-8. +- Sort input files deterministically. +- Skip hidden files. +- Skip `.txt`, binary, temp, and log files. +- Print concise progress logs. +- Produce exactly these compact package files: + - `KB__00_INDEX.md` + - `KB__01_NAVIGATION.md` + - `KB__02_CONTENT.md` + - `KB__03_WORKFLOWS_TRACEABILITY.md` + - `KB__04_SMOKE_QA.md` + - `README.md` + - `MANIFEST.md` +- `KB__00_INDEX.md` must include package purpose, generation timestamp, source folders used, output files, recommended ChatGPT Project usage, limitations, and update procedure. +- `KB__01_NAVIGATION.md` must merge: + - `publish/chatgpt_project/INDEX.md` + - `publish/chatgpt_project/CONCEPT_MAP.md` + - `publish/chatgpt_project/KB_USAGE_GUIDE.md` +- `KB__01_NAVIGATION.md` must preserve headings, add a source file marker before each merged section, remove duplicated blank lines, and avoid semantic rewrites. +- `KB__02_CONTENT.md` must merge markdown files from `publish/markdown_kb/`, sorted alphabetically. +- `KB__02_CONTENT.md` must preserve original relative path before each section and preserve markdown headings. +- `KB__02_CONTENT.md` must exclude files larger than the configured max size and list skipped files in `MANIFEST.md`. +- `KB__02_CONTENT.md` must not include binary files or `.txt` files. +- `KB__03_WORKFLOWS_TRACEABILITY.md` must merge: + - `publish/chatgpt_project/WORKFLOW_MAP.md` + - `publish/chatgpt_project/TRACEABILITY_GUIDE.md` + - sections from `publish/chatgpt_project/AI_KB_Context_File_v1.0.md` related to Publish Metadata, Knowledge Index, and Source Traceability. +- If section extraction from `AI_KB_Context_File_v1.0.md` is unreliable, include only the first 300 lines and record the limitation in `MANIFEST.md`. +- `KB__04_SMOKE_QA.md` must merge `publish/chatgpt_project/SMOKE_QUESTIONS.md` and add recommended smoke test sequence, pass/fail checklist, and guidance for reporting retrieved chunks back to ChatGPT. +- `README.md` must explain what the compact package is, how to upload it into ChatGPT Project, how to use it together with Open WebUI, what not to upload, and the update command. +- `MANIFEST.md` must include generation timestamp, included files, skipped files, file sizes, warnings, and validation status. + +## Constraints + +- Minimal changes only. +- Do not refactor unrelated project files. +- Do not invent missing files. +- If an expected source folder is missing, fail with a clear error. +- Do not delete existing source files. +- Do not move original files. +- Do not modify raw input files. +- Do not use: + - `workspace/raw/` + - `workspace/clean_notes/` + - `workspace/source_cards/` + - `workspace/chunks/` + - `temp/` + - `logs/` +- Do not use LLM calls. +- Do not use Ollama. +- Do not use network calls. +- Do not create embeddings. +- Do not add a vector database. +- Do not add dependencies. + +## Acceptance criteria + +- `publish/chatgpt_project_compact/` exists after generation. +- All seven required compact package files exist. +- Compact output files are UTF-8 readable. +- No `.txt` files are included in compact content. +- Raw transcripts are not included. +- Source cards are not included. +- `MANIFEST.md` lists included and skipped files. +- `README.md` has upload instructions. +- Script can be rerun without errors. +- Validation commands complete: + - `python3 scripts/build_chatgpt_compact_kb.py` + - `ls -lah publish/chatgpt_project_compact` + - `find publish/chatgpt_project_compact -type f` + +## Risks + +- `publish/markdown_kb/` currently contains only one observed markdown file, so compact content may be smaller than expected. +- The requested `AI_KB_Context_File_v1.0.md` path is present under `publish/chatgpt_project/`, not at the project root. +- Section extraction from `AI_KB_Context_File_v1.0.md` may be unreliable if headings do not match the requested section names exactly. +- Large markdown files may be skipped by the default `1 MB` limit and must be visible in `MANIFEST.md`. diff --git a/SPEC_synthesis_layer.md b/SPEC_synthesis_layer.md new file mode 100644 index 0000000..2f16a49 --- /dev/null +++ b/SPEC_synthesis_layer.md @@ -0,0 +1,112 @@ +# SPEC + +## Goal + +Add a Knowledge Distillation / Synthesis Layer to the existing Knowledge Base compact package. + +The layer should move the KB beyond navigation, routing, workflow, traceability, and placeholder pages by creating deterministic files for distilled concepts, operational frameworks, patterns, anti-patterns, and practical use cases. When reliable synthesis is not possible without LLM calls, the output must use explicit TODO blocks, source references, confidence labels, and clear gaps instead of fabricated knowledge. + +## Current state + +- Project root is `/Users/sst/Documents/Python Progect/[Транскрибация] Knoweledge Base/`. +- Root `SPEC.md` describes a different MVP-10 lightweight search CLI task and must not be overwritten by this spec. +- Existing compact ChatGPT Project package files are present under `publish/chatgpt_project_compact/`. +- Source folders exist or have observed files: + - `workspace/source_cards/` + - `workspace/clean_notes/` + - `publish/markdown_kb/` + - `publish/chatgpt_project/` +- Existing compact files must not be deleted. +- `scripts/build_synthesis_layer.py` does not currently exist. + +## Requirements + +- Add `scripts/build_synthesis_layer.py`. +- The script must run from project root: + - `python3 scripts/build_synthesis_layer.py` +- Optional CLI flags: + - `--max-sources-per-concept 20` + - `--output publish/chatgpt_project_compact` +- Use only Python standard library by default. +- Default mode must work without LLM calls. +- If optional `--use-llm` is implemented, it must be optional and must fall back to deterministic skeleton output on failure. +- Read sources in this priority order: + - `workspace/source_cards/` + - `workspace/clean_notes/` + - `publish/markdown_kb/` + - `publish/chatgpt_project/` +- Do not use: + - `workspace/raw/` + - `workspace/chunks/` + - `logs/` + - `temp/` + - `embeddings/` + - `vector_db/` +- Create or update only these new output files under `publish/chatgpt_project_compact/`: + - `KB__05_CANONICAL_CONCEPTS.md` + - `KB__06_OPERATIONAL_FRAMEWORKS.md` + - `KB__07_PATTERNS_AND_FAILURES.md` + - `KB__08_USE_CASES_FOR_SERGEY.md` + - `SYNTHESIS_MANIFEST.md` +- Do not delete or overwrite existing compact files except these five synthesis-layer outputs. +- Extract candidate concepts from headings, filenames, and repeated terms. +- Generate deterministic markdown skeletons with source references and confidence labels. +- Every synthesized block must separate: + - Facts + - Interpretation + - Operational use + - Limitations + - Source evidence + - Confidence +- When source support is weak, write `Evidence is weak / not enough source support.` +- Each concept, framework, pattern, anti-pattern, and use case must include source evidence or explicitly mark weak/topic-level evidence. +- `KB__05_CANONICAL_CONCEPTS.md` must contain 20-50 canonical concepts and include all minimum required concepts from the task request. +- `KB__06_OPERATIONAL_FRAMEWORKS.md` must contain 10-20 reusable frameworks and include all minimum required frameworks from the task request. +- `KB__07_PATTERNS_AND_FAILURES.md` must include `# Patterns` and `# Anti-patterns / Failure Modes` and include all minimum required patterns and anti-patterns from the task request. +- `KB__08_USE_CASES_FOR_SERGEY.md` must include all ten minimum required Sergey use cases from the task request. +- `SYNTHESIS_MANIFEST.md` must include generation timestamp, inputs used, outputs created, concepts created, frameworks created, patterns created, anti-patterns created, evidence quality summary, warnings, and validation status. +- Print concise progress logs. +- Preserve UTF-8. +- Do not modify raw sources. +- Do not move or delete source files. + +## Constraints + +- Minimal changes only. +- Do not refactor unrelated files. +- Do not invent missing sources. +- Do not claim semantic synthesis is complete if evidence is weak. +- Prefer explicit TODO blocks over hallucinated knowledge. +- Keep output useful for ChatGPT Project upload. +- Do not copy raw transcripts into output. +- Do not use embeddings or vector databases. +- Do not use network calls by default. +- Do not add dependencies. +- Python and SQL must perform numeric calculations if any are needed; no numeric calculations are expected for this task. + +## Acceptance criteria + +- `scripts/build_synthesis_layer.py` exists. +- Running `python3 scripts/build_synthesis_layer.py` succeeds from project root. +- All five new synthesis output files exist under `publish/chatgpt_project_compact/`. +- Output files are UTF-8 readable. +- Every concept has evidence and confidence. +- Every framework has trigger, inputs, ordered steps, outputs, and QA gates. +- Anti-patterns include `Inventing definitions where source says not found`. +- No raw transcripts are copied into output. +- No source files are deleted or moved. +- `SYNTHESIS_MANIFEST.md` includes warnings and evidence quality summary. +- Script can be rerun without errors. +- Required validation commands complete: + - `python3 scripts/build_synthesis_layer.py` + - `ls -lah publish/chatgpt_project_compact` + - `find publish/chatgpt_project_compact -maxdepth 1 -type f | sort` + - `python3 -m py_compile scripts/build_synthesis_layer.py` + +## Risks + +- High-quality synthesis may not be safely automatable without LLM calls. +- Deterministic output may be more of a structured synthesis skeleton than polished distilled knowledge. +- Source-card and clean-note evidence may be noisy or unevenly distributed across concepts. +- Some required concepts, frameworks, patterns, or use cases may have weak or topic-level evidence only. +- Adding optional LLM mode may introduce validation complexity and should not be required for default execution. diff --git a/SPEC_transcript_to_kb_pipeline.md b/SPEC_transcript_to_kb_pipeline.md new file mode 100644 index 0000000..1cdc908 --- /dev/null +++ b/SPEC_transcript_to_kb_pipeline.md @@ -0,0 +1,1271 @@ +# SPEC.md — transcript_to_kb_pipeline + +Version: v0.2 +Status: active candidate +Current design: transcript-only first +Current date: 2026-05-12 +Primary input: `*.txt` transcript files +Optional legacy input: `*.md` notes files + +--- + +## 1. Purpose + +Create a standalone project: + +```text +transcript_to_kb_pipeline/ +``` + +that converts ready-made transcript files into a structured knowledge base. + +Main pipeline: + +```text +raw .txt +→ inventory +→ chunking +→ clean note +→ source card +→ knowledge base +→ judge reports +→ publish +``` + +Core rule: + +```text +.txt = source of truth +.md = optional legacy draft +``` + +The project does not download videos and does not perform audio transcription. It processes already available transcript files. + +--- + +## 2. Design Decision + +Previous design assumed paired files: + +```text +.txt transcript + .md notes +``` + +Updated design: + +```text +.txt transcript only +``` + +Rationale: + +- A `.txt` transcript already contains the source material needed for clean notes, source cards, and KB build. +- A `.md` note may be useful if already available, but it may also contain generation artifacts, contradictions, or hallucinated summaries. +- Therefore, `.md` is treated as an optional legacy draft, never as source of truth. + +--- + +## 3. Modes + +### Mode A — Default: transcript-only + +```text +input/raw/*.txt +→ inventory +→ chunking +→ clean notes +→ source cards +→ knowledge base +→ judge +``` + +Use this mode for new processing. + +### Mode B — Legacy: transcript + existing notes + +```text +input/raw/*.txt +input/legacy_notes/*.md +→ compare md vs txt +→ clean legacy note +→ mark issues +``` + +Use only when old `.md` notes already exist and need reuse or review. + +### Mode C — Repair mode + +```text +source_id +→ rebuild clean note / source card / judge report +``` + +Use when a specific artifact is poor or marked `review_required`. + +--- + +## 4. Non-goals + +Do not implement in the base project unless explicitly requested: + +```text +- video downloading +- audio transcription +- mutation of external transcription project +- mutation of external downloads/ +- mutation of external output/ +- treating .md as source of truth +- embeddings / vector DB in MVP +- web UI in MVP +- scheduler in MVP +``` + +--- + +## 5. Target Project Structure + +```text +transcript_to_kb_pipeline/ + README.md + SPEC.md + AGENTS.md + TASKS.md + CHANGELOG.md + requirements.txt + .gitignore + + input/ + raw/ + *.txt + + legacy_notes/ + *.md + + workspace/ + inventory/ + files_index.csv + sources_index.csv + + chunks/ + / + chunk_001.txt + chunk_002.txt + chunk_manifest.csv + + clean_notes/ + .clean.md + + source_cards/ + .source_card.md + + knowledge/ + topics/ + concepts/ + indexes/ + INDEX.md + sources_index.csv + concepts_index.csv + reports/ + quality_report.md + judge_report.md + review_queue.md + conflicts.md + unsupported_claims.md + + publish/ + chatgpt_project/ + obsidian/ + markdown_kb/ + + prompts/ + clean_note_ru.md + source_card_ru.md + kb_build_ru.md + judge_ru.md + + scripts/ + run_inventory.py + run_chunking.py + run_clean_note.py + run_source_card.py + run_kb_build.py + run_judge.py + run_publish.py + + src/ + transcript_to_kb/ + __init__.py + inventory.py + chunking.py + llm_client.py + clean_note.py + source_card.py + kb_build.py + judge.py + schemas.py + paths.py + errors.py + + tests/ + fixtures/ + raw/ + legacy_notes/ + expected/ +``` + +--- + +## 6. Artifact Roles + +| Artifact | Role | Source of truth | +|---|---|---| +| `input/raw/*.txt` | raw transcript | yes | +| `input/legacy_notes/*.md` | old notes / optional draft | no | +| `files_index.csv` | file inventory | yes | +| `sources_index.csv` | source registry | yes | +| `chunks/` | transcript chunks for LLM | derived | +| `clean_notes/*.clean.md` | clean note | derived | +| `source_cards/*.source_card.md` | normalized source card | yes after judge | +| `knowledge/topics/` | topic synthesis | derived | +| `knowledge/concepts/` | concept synthesis | derived | +| `knowledge/reports/` | quality control | yes | +| `publish/` | final export | derived | + +--- + +## 7. LLM Usage Policy + +### No LLM + +Use Python/deterministic processing for: + +```text +Phase 1 — Inventory +Phase 2 — Chunking +``` + +### LLM allowed + +Use LLM for: + +```text +Phase 3 — Clean Note +Phase 4 — Source Card +Phase 5 — KB Build +Phase 6 — Judge +``` + +Main rule: + +```text +Python prepares clean input. +LLM extracts and structures meaning. +Judge validates output quality. +``` + +--- + +## 8. Phase 0 — Bootstrap + +### Goal + +Create the project and base contracts. + +### Files + +```text +README.md +SPEC.md +AGENTS.md +TASKS.md +CHANGELOG.md +requirements.txt +.gitignore +``` + +### Acceptance Criteria + +```text +- project structure exists +- raw input is separated from workspace +- generated artifacts are not committed +- README explains transcript-only mode +- AGENTS.md forbids raw file mutation +``` + +--- + +## 9. Phase 1 — Transcript Inventory + +### Goal + +Create an inventory of all raw `.txt` transcript files. + +### Input + +```text +input/raw/*.txt +``` + +### Command + +```bash +python3 scripts/run_inventory.py --input input/raw +``` + +### Output + +```text +workspace/inventory/files_index.csv +workspace/inventory/sources_index.csv +``` + +### files_index.csv + +```csv +file_id, +source_id, +filename, +extension, +base_name, +date_prefix, +slug, +size_bytes, +line_count, +char_count, +language_hint, +status, +path +``` + +### Source ID Contract + +```text +source_id = sha256(base_name.lower().encode("utf-8")).hexdigest()[:12] +``` + +### Acceptance Criteria + +```text +- all .txt files are indexed +- source_id is stable +- files_index.csv is created +- sources_index.csv is created +- raw files are unchanged +``` + +--- + +## 10. Phase 2 — Transcript Chunking + +### Goal + +Split long transcripts into chunks suitable for LLM processing. + +### Command + +```bash +python3 scripts/run_chunking.py --input input/raw --chunk-size 6000 +``` + +### Output + +```text +workspace/chunks// + chunk_001.txt + chunk_002.txt + chunk_manifest.csv +``` + +### chunk_manifest.csv + +```csv +source_id, +chunk_id, +chunk_path, +start_char, +end_char, +char_count, +line_start, +line_end +``` + +### Rules + +```text +- do not mutate raw transcript +- preserve chunk order +- avoid splitting in the middle of a line when possible +- every chunk must reference source_id +``` + +### Acceptance Criteria + +```text +- chunks are created +- chunk_manifest.csv is created +- chunk order is preserved +- raw files are unchanged +``` + +--- + +## 11. Phase 3 — LLM Clean Note + +### Goal + +Generate a clean note from transcript chunks. + +### Command + +```bash +python3 scripts/run_clean_note.py --source-id +``` + +Batch mode: + +```bash +python3 scripts/run_clean_note.py --all +``` + +### Input + +```text +workspace/chunks//*.txt +``` + +### Output + +```text +workspace/clean_notes/.clean.md +``` + +### Default Model Strategy + +```text +Ollama / local LLM — batch clean note drafts +ChatGPT Reasoning — high-value sources or difficult synthesis +``` + +### Clean Note Template + +```markdown +# Clean Note + +## Metadata +- source_id: +- title: +- source_file: +- language: +- processing_mode: +- model: +- prompt_version: +- review_required: + +## Short Summary +... + +## Main Ideas +1. ... +2. ... +3. ... + +## Procedures / Workflow +... + +## Concepts +... + +## Practical Rules +... + +## Risks / Caveats +... + +## Evidence Pointers +- chunk_001: +- chunk_002: + +## Open Questions +... + +## Review Notes +... +``` + +### Prompt Constraints + +```text +- Work only from transcript chunks. +- Do not add external knowledge. +- Do not invent facts. +- If data is absent, write "not found in source". +- Preserve practical procedures. +- Add evidence pointers by chunk. +``` + +### Acceptance Criteria + +```text +- clean note is created +- no blocks +- metadata exists +- evidence pointers exist +- no external facts added +- doubtful sections are marked review_required +``` + +--- + +## 12. Phase 4 — Source Card + +### Goal + +Create one normalized source card per transcript. + +### Command + +```bash +python3 scripts/run_source_card.py --source-id +``` + +### Input + +```text +input/raw/.txt +workspace/clean_notes/.clean.md +workspace/chunks// +``` + +### Output + +```text +workspace/source_cards/.source_card.md +``` + +### Source Card Template + +```markdown +# Source Card + +## Metadata +- source_id: +- title: +- date_prefix: +- raw_transcript_path: +- clean_note_path: +- chunk_manifest_path: +- model: +- prompt_version: +- confidence: +- review_required: + +## Core Topic +... + +## Key Concepts +| Concept | Definition | Evidence | Confidence | +|---|---|---|---| + +## Procedures / Workflows +| Procedure | Steps | Evidence | Caveats | +|---|---|---|---| + +## Practical Rules +| Rule | When to use | Evidence | Risk | +|---|---|---|---| + +## Examples +... + +## Risks / Caveats +... + +## Not Found / Unclear +... + +## Tags +... + +## Human Review +- required: +- reason: +``` + +### Acceptance Criteria + +```text +- source card is created +- every important claim has evidence +- unsupported claims are marked +- no external facts added +- review_required works +``` + +--- + +## 13. Phase 5 — Knowledge Base Build + +### Goal + +Build topic and concept pages from source cards. + +### Command + +```bash +python3 scripts/run_kb_build.py --all +``` + +By topic: + +```bash +python3 scripts/run_kb_build.py --topic "File Organization" +``` + +### Input + +```text +workspace/source_cards/*.source_card.md +``` + +### Output + +```text +workspace/knowledge/ + topics/ + concepts/ + indexes/ +``` + +### Example concept pages + +```text +concepts/ + para_method.md + active_projects.md + areas.md + resources.md + archive.md + weekly_review.md + file_organization_workflow.md +``` + +### Topic Page Template + +```markdown +# [Topic Name] + +## What this topic covers +... + +## Core Ideas +... + +## Concepts +- [[concept_1]] +- [[concept_2]] + +## Source-backed Rules +... + +## Source Cards +- + +## Conflicts / Caveats +... + +## Review Status +... +``` + +### Acceptance Criteria + +```text +- topic pages are created +- concept pages are created +- INDEX.md is created +- sources_index.csv is created +- concepts_index.csv is created +- important claims link to source cards +``` + +--- + +## 14. Phase 6 — Judge / Quality Review + +### Goal + +Validate clean notes, source cards, and KB pages. + +### Command + +```bash +python3 scripts/run_judge.py --all +``` + +### Output + +```text +workspace/knowledge/reports/ + judge_report.md + review_queue.md + conflicts.md + unsupported_claims.md +``` + +### Judge Checks + +```text +- clean note contains no +- source card contains no unsupported claims +- KB topic page links to source cards +- no empty concepts +- no broken links +- no unresolved contradictions +- review_required sources are not published as clean +``` + +### Acceptance Criteria + +```text +- judge_report.md is created +- review_queue.md is created +- conflicts.md is created +- unsupported_claims.md is created +- every issue has source_id +``` + +--- + +## 15. Phase 7 — Publish + +### Goal + +Prepare final export. + +### Output + +```text +publish/ + chatgpt_project/ + obsidian/ + markdown_kb/ +``` + +### ChatGPT Project mode + +```text +publish/chatgpt_project/ + AI_[topic]_Context_File_v1.0.md + INDEX.md +``` + +### Obsidian mode + +```text +publish/obsidian/ + topics/ + concepts/ + sources/ + INDEX.md +``` + +### Markdown KB mode + +```text +publish/markdown_kb/ + full_kb.md + sources_index.csv + concepts_index.csv +``` + +### Acceptance Criteria + +```text +- publish folder is created +- target format is selected +- raw files are not included unless explicitly requested +- source traceability is preserved +``` + +--- + +## 16. Legacy Mode — `.txt + .md` + +### Purpose + +Use when old `.md` notes already exist. + +### Pipeline + +```text +.txt source of truth +.md optional draft +→ compare +→ clean legacy note +→ mark issues +``` + +### Checks + +```text +- blocks +- contradictory summary blocks +- "no information" claims when transcript contains information +- invented claims +- missing important procedures +``` + +### Output + +```text +workspace/legacy_review/ + legacy_notes_report.md + legacy_clean_notes/ +``` + +### Rule + +```text +Legacy .md never overrides transcript. +``` + +--- + +## 17. Model / Tool Selection + +### Python + +Use for: + +```text +- inventory +- chunking +- indexes +- manifests +- link checking +- file validation +``` + +### Ollama / local LLM + +Use for: + +```text +- draft clean notes +- draft source cards +- batch processing +``` + +### ChatGPT Reasoning + +Use for: + +```text +- KB build +- difficult synthesis +- judge review +- high-value sources +``` + +### Codex + +Use for: + +```text +- scripts +- tests +- CLI +- repository structure +- validation +``` + +--- + +## 18. .gitignore + +```gitignore +workspace/* +publish/* + +!workspace/**/.gitkeep +!publish/**/.gitkeep + +__pycache__/ +.pytest_cache/ +.env +.venv/ +.DS_Store +*.log +``` + +--- + +## 19. requirements.txt + +MVP: + +```txt +pytest>=8.0 +PyYAML>=6.0 +``` + +LLM phase later: + +```txt +pydantic>=2.0 +``` + +--- + +## 20. AGENTS.md Requirements + +```markdown +# AGENTS.md + +## Scope + +Current project: transcript_to_kb_pipeline. + +Default input: +- raw .txt transcripts + +Optional legacy input: +- .md notes + +## Do Not Touch + +- raw input files +- external transcription project +- external output/ +- external downloads/ + +## Rules + +- .txt is source of truth. +- .md is optional legacy draft. +- Do not modify raw files. +- Do not use LLM in inventory/chunking. +- Do not invent content. +- Every important claim must have evidence. +- Prefer review_queue over silent failure. + +## Implementation Order + +1. Inventory +2. Chunking +3. Clean note generation +4. Source card generation +5. KB build +6. Judge +7. Publish + +## Testing + +Run tests before final response. +``` + +--- + +## 21. TASKS.md Requirements + +```markdown +# TASKS.md + +## MVP-1 — Transcript Inventory + Chunking + +- [ ] Bootstrap project +- [ ] Add README.md +- [ ] Add SPEC.md +- [ ] Add AGENTS.md +- [ ] Add requirements.txt +- [ ] Add .gitignore +- [ ] Implement inventory +- [ ] Implement source_id +- [ ] Implement chunking +- [ ] Add tests +- [ ] Run tests + +## MVP-2 — Clean Notes + +- [ ] Add clean_note prompt +- [ ] Add LLM client interface +- [ ] Generate clean notes +- [ ] Add evidence pointers +- [ ] Add review_required logic +- [ ] Add tests / fixtures + +## MVP-3 — Source Cards + +- [ ] Add source card prompt +- [ ] Generate source cards +- [ ] Validate evidence fields +- [ ] Add sources index + +## MVP-4 — KB Build + +- [ ] Generate topic pages +- [ ] Generate concept pages +- [ ] Update INDEX.md +- [ ] Generate concepts_index.csv + +## MVP-5 — Judge + +- [ ] Generate judge_report.md +- [ ] Generate conflicts.md +- [ ] Generate unsupported_claims.md +- [ ] Update review_queue.md +``` + +--- + +## 22. Required Tests + +```text +test_inventory.py +test_source_id.py +test_chunking.py +test_clean_note_schema.py +test_source_card_schema.py +test_kb_links.py +test_judge_report.py +test_raw_files_immutable.py +``` + +--- + +## 23. Fixtures + +```text +tests/fixtures/raw/ + para_sample.txt + ap_antifraud_sample.txt + short_transcript.txt + long_transcript.txt + mixed_language_transcript.txt + empty_transcript.txt +``` + +Legacy mode fixtures: + +```text +tests/fixtures/legacy_notes/ + note_with_think.md + contradictory_note.md + good_note.md +``` + +--- + +## 24. Acceptance Criteria + +### MVP-1 is ready when: + +```text +1. project structure created +2. inventory works +3. source_id stable +4. chunking works +5. raw files unchanged +6. files_index.csv created +7. sources_index.csv created +8. chunk_manifest.csv created +9. tests pass +``` + +### MVP-2 is ready when: + +```text +1. clean notes created +2. LLM prompt version tracked +3. model name tracked +4. evidence pointers present +5. no +6. review_required works +``` + +### MVP-3 is ready when: + +```text +1. source cards created +2. concepts extracted +3. procedures extracted +4. every key claim has evidence +5. unsupported claims marked +``` + +### MVP-4 is ready when: + +```text +1. topics created +2. concepts created +3. indexes created +4. source traceability preserved +``` + +### MVP-5 is ready when: + +```text +1. judge report created +2. conflicts detected +3. unsupported claims listed +4. review queue updated +``` + +--- + +## 25. IMPLEMENTATION_TASK_001 + +### Goal + +Create MVP-1 for `transcript_to_kb_pipeline`: + +```text +Bootstrap → Inventory → Chunking +``` + +### Scope + +Implement only: + +```text +- project structure +- inventory +- source_id +- chunking +- indexes +- tests +``` + +### Non-goals + +Do not implement: + +```text +- LLM calls +- clean note generation +- source cards +- KB build +- judge +- publish +- legacy .md review +``` + +### Commands + +```bash +python3 scripts/run_inventory.py --input input/raw +python3 scripts/run_chunking.py --input input/raw --chunk-size 6000 +python3 -m pytest +``` + +### Acceptance Criteria + +```text +- files_index.csv created +- sources_index.csv created +- chunks created +- chunk_manifest.csv created +- source_id stable +- raw files unchanged +- tests pass +``` + +--- + +## 26. IMPLEMENTATION_TASK_002 + +### Goal + +Add LLM clean note generation from transcript chunks. + +### Scope + +Implement: + +```text +- prompts/clean_note_ru.md +- llm_client interface +- run_clean_note.py +- clean note output contract +- evidence pointers +- review_required logic +``` + +### Non-goals + +Do not implement: + +```text +- source cards +- KB build +- publish +``` + +### Acceptance Criteria + +```text +- clean notes generated +- prompt version tracked +- model name tracked +- evidence pointers present +- raw files unchanged +``` + +--- + +## 27. Risks + +| Risk | Consequence | Mitigation | +|---|---|---| +| LLM used too early | LLM prettifies chaos | Inventory/chunking first | +| `.md` treated as source | legacy noise enters KB | `.txt = source of truth` | +| no evidence | KB becomes unsupported summary | evidence pointers required | +| no Judge | hallucinations get published | Judge phase required | +| unstable source_id | links break | hash from base_name | +| no tests | fragile pipeline | pytest fixtures | +| too many phases at once | Codex scope creep | implementation tasks one by one | + +--- + +## 28. Final Formula + +```text +One .txt +→ Clean Note +→ Source Card + +Many Source Cards +→ Knowledge Base + +Knowledge Base +→ Judge +→ Publish +``` + +Main principle: + +```text +LLM does not replace order. +LLM works only after Python prepares clean input. +``` + +--- + +## 29. Next Step + +Next: Codex / AI Operator + +Do: + +```text +IMPLEMENTATION_TASK_001: +Bootstrap → Inventory → Chunking +``` + +Why: + +```text +First, create the technical foundation: +source_id, indexes, chunks, raw immutability, tests. +``` diff --git a/SPEC_v1.md b/SPEC_v1.md new file mode 100644 index 0000000..7d750d1 --- /dev/null +++ b/SPEC_v1.md @@ -0,0 +1,645 @@ +# SPEC: Stage 1 Data Contract Preparation + +Version: v01 +Status: ready for Codex +Scope: read-only preparation before Stage 1 ETL +Project area: finance data processing / Excel raw files + +--- + +## 1. Goal + +Prepare a safe **Stage 1 Data Contract Package** for loading and normalizing Excel files from `data/raw`. + +The goal is not to build final analytics yet. +The goal is to produce a reproducible preparation layer that allows the user to approve: + +- authoritative files; +- period rules; +- currency rules; +- plan / fact rules; +- DDS mapping; +- INOUT mapping; +- target Stage 1 schema. + +Target flow: + +```text +data/raw +→ file inventory +→ schema groups +→ column mapping draft +→ business-rule detection reports +→ decisions needed +→ approved Stage 1 contract +``` + +--- + +## 2. Background + +A previous read-only reconnaissance found **57 Excel files** in `data/raw`. + +Detected schema groups: + +| Group | Column count | File count | Categories | +|---|---:|---:|---| +| A | 52 | 41 | `budget_rows`, `dds`, `p-fact` | +| B | 30 | 10 | `dds` | +| C | 38 | 5 | `budget_rows`, `dds` | +| D | 19 | 1 | `cons_budget` | + +Main blocker: + +```text +Stage 1 cannot be safely designed until authoritative files, period rules, +currency rules, plan/fact rules, DDS, INOUT, and target schema are confirmed. +``` + +--- + +## 3. Current State + +Known facts: + +- `data/raw` exists. +- 57 Excel files were found. +- Files have at least 4 different schema groups. +- Some files appear to belong to `budget_rows`, `dds`, `p-fact`, and `cons_budget` groups. +- Existing ETL, stage/mart/report layers and raw files must not be changed during this preparation task. + +Unknown / needs confirmation: + +- Which files are authoritative. +- Whether there are duplicates, historical versions, exports, or test files. +- How period should be derived. +- How currency should be derived. +- How to classify plan / fact / forecast. +- What DDS values mean. +- What INOUT values mean. +- What final Stage 1 schema should be approved. + +--- + +## 4. Target Behavior + +Codex must create a read-only Stage 1 preparation package under: + +```text +out/stage1/ +``` + +The package must contain inventories, mapping drafts, detection reports, and a decision file for the user. + +Codex must not silently infer business meaning. +If something is ambiguous, mark it as: + +```text +unknown +needs_confirmation +ambiguous +``` + +--- + +## 5. Non-goals + +Codex must not: + +- modify files in `data/raw`; +- delete, move, or rename raw files; +- build final analytics; +- build mart/report layer; +- convert currencies; +- merge plan/fact data without confirmation; +- infer period silently; +- infer DDS or INOUT business meaning silently; +- refactor unrelated ETL code; +- rewrite the project architecture; +- make destructive operations. + +--- + +## 6. Required Outputs + +Codex must create the following files: + +```text +out/stage1/file_inventory.csv +out/stage1/schema_groups.md +out/stage1/authoritative_candidates.md +out/stage1/column_mapping_draft.xlsx +out/stage1/column_mapping_draft.md +out/stage1/period_detection_report.md +out/stage1/currency_detection_report.md +out/stage1/plan_fact_detection_report.md +out/stage1/dds_values_report.xlsx +out/stage1/inout_values_report.xlsx +out/stage1/dds_inout_mapping_draft.md +out/stage1/stage1_decisions_needed.md +``` + +Optional, if useful: + +```text +out/stage1/processing_log.md +out/stage1/stage1_schema_draft.md +``` + +--- + +## 7. File Inventory Requirements + +Create: + +```text +out/stage1/file_inventory.csv +``` + +Required columns: + +| Column | Description | +|---|---| +| `file_path` | Full path to source file | +| `file_name` | File name | +| `source_group` | `budget_rows` / `dds` / `p-fact` / `cons_budget` / `unknown` | +| `sheet_names` | List of sheets | +| `sheet_count` | Number of sheets | +| `column_count` | Number of columns in primary detected sheet | +| `row_count` | Number of rows | +| `schema_hash` | Reproducible hash of column structure | +| `period_candidate` | Period if detected | +| `currency_candidate` | Currency if detected | +| `scenario_candidate` | `plan` / `fact` / `forecast` / `unknown` | +| `is_authoritative_candidate` | `yes` / `no` / `unknown` | +| `reason` | Why the file is or is not a candidate | +| `warnings` | Any issues found | + +--- + +## 8. Schema Group Report + +Create: + +```text +out/stage1/schema_groups.md +``` + +For each schema group, include: + +- schema hash; +- number of files; +- column count; +- full column list; +- files in group; +- example rows if safe and useful; +- detected source categories; +- risks and ambiguities. + +--- + +## 9. Authoritative Candidate Report + +Create: + +```text +out/stage1/authoritative_candidates.md +``` + +For each candidate group, explain: + +- which files may be authoritative; +- which files may be duplicates or historical versions; +- evidence for the recommendation; +- what requires user confirmation. + +Do not make a final decision without explicit confirmation. + +--- + +## 10. Column Mapping Draft + +Create: + +```text +out/stage1/column_mapping_draft.xlsx +out/stage1/column_mapping_draft.md +``` + +For each original column in each schema group, include: + +| Column | Description | +|---|---| +| `schema_hash` | Schema group identifier | +| `source_group` | Candidate source group | +| `original_column` | Original column name | +| `normalized_column_candidate` | Proposed normalized name | +| `data_type_candidate` | string / date / decimal / integer / boolean / unknown | +| `example_values` | Representative examples | +| `null_share` | Share of empty values | +| `unique_count` | Number of unique values | +| `notes` | Observations | +| `needs_user_confirmation` | yes / no | + +Rules: + +- Do not rename columns in raw files. +- Mapping is a draft only. +- If meaning is unclear, mark `needs_user_confirmation = yes`. + +--- + +## 11. Period Detection Report + +Create: + +```text +out/stage1/period_detection_report.md +``` + +Check possible period sources: + +| Source | Examples | +|---|---| +| file name | `2026-04`, `04.2026`, `Apr`, `April` | +| sheet name | `Апрель`, `2026-04` | +| separate column | `period`, `month`, `date` | +| operation date | transaction date / created date | +| unavailable | `unknown` | + +Report: + +- detected candidates; +- conflicts; +- recommended rule; +- required user confirmation. + +Rule: + +```text +If period candidates conflict, do not choose automatically. +``` + +--- + +## 12. Currency Detection Report + +Create: + +```text +out/stage1/currency_detection_report.md +``` + +Check possible currency sources: + +| Source | Examples | +|---|---| +| separate column | `currency`, `валюта` | +| file name | `EUR`, `USD`, `RUB` | +| sheet name | currency marker | +| unavailable | `unknown` | + +Rules: + +- Do not convert currencies on Stage 1. +- Preserve original currency if available. +- Use `unknown` if unavailable. +- Record conflicts and ambiguities. + +--- + +## 13. Plan / Fact Detection Report + +Create: + +```text +out/stage1/plan_fact_detection_report.md +``` + +Classify rows or files as one of: + +```text +plan +fact +forecast +unknown +``` + +Check possible sources: + +| Source | Examples | +|---|---| +| file name | `plan`, `fact`, `p-fact` | +| folder name | `budget_rows`, `p-fact` | +| sheet name | `План`, `Факт` | +| column value | `scenario`, `type`, `version` | + +Rule: + +```text +If scenario is not obvious, set it to unknown. +``` + +--- + +## 14. DDS and INOUT Mapping Draft + +Create: + +```text +out/stage1/dds_values_report.xlsx +out/stage1/inout_values_report.xlsx +out/stage1/dds_inout_mapping_draft.md +``` + +Extract unique values for candidate DDS and INOUT columns. + +Required fields: + +| Column | Description | +|---|---| +| `source_file` | Source file path | +| `source_sheet` | Sheet name | +| `column_name` | Column containing DDS or INOUT candidate | +| `unique_value` | Raw unique value | +| `count` | Number of occurrences | +| `example_rows` | Example row numbers or compact examples | +| `normalized_candidate` | Proposed normalized value, if safe | +| `needs_confirmation` | yes / no | +| `notes` | Ambiguities and comments | + +Rules: + +- Do not silently normalize ambiguous values. +- Preserve raw values. +- User must approve final DDS and INOUT mapping. + +--- + +## 15. Draft Stage 1 Schema + +Prepare a draft schema for: + +```text +stage_finance_rows +``` + +Suggested fields: + +| Field | Type | Description | +|---|---|---| +| `source_file` | string | Source file path | +| `source_sheet` | string | Source sheet | +| `source_row_number` | integer | Source row number | +| `source_group` | string | File/source group | +| `period` | string/date | Reporting period | +| `operation_date` | date/null | Operation date if available | +| `scenario` | string | plan / fact / forecast / unknown | +| `currency` | string | Currency code or unknown | +| `amount` | decimal/null | Parsed amount | +| `amount_original` | string | Original amount value | +| `dds_raw` | string/null | Original DDS value | +| `dds_normalized` | string/null | Normalized DDS value | +| `inout_raw` | string/null | Original INOUT value | +| `inout_normalized` | string/null | Normalized INOUT value | +| `counterparty` | string/null | Counterparty if available | +| `project` | string/null | Project / article / direction if available | +| `description` | string/null | Description | +| `raw_payload_json` | json/string | Full source row as JSON | +| `load_timestamp` | datetime | Load timestamp | +| `validation_status` | string | ok / warning / error | +| `validation_notes` | string | Validation comments | + +This is a draft only. +Do not finalize without user approval. + +--- + +## 16. Decisions Needed File + +Create: + +```text +out/stage1/stage1_decisions_needed.md +``` + +Required structure: + +```markdown +# Stage 1 Decisions Needed + +## 1. Authoritative files + +### Question +Which files should be treated as authoritative sources? + +### Options +- Use all 57 files +- Use only latest versions +- Use only specific folders +- Exclude duplicates / archives / test files + +### Evidence +... + +### Recommendation +... + +### User decision +TBD + +--- + +## 2. Period rule +... + +## 3. Currency rule +... + +## 4. Plan / Fact rule +... + +## 5. DDS rule +... + +## 6. INOUT rule +... + +## 7. Target Stage 1 schema approval +... +``` + +This file is the most important output of the task. + +--- + +## 17. Implementation Requirements + +If Codex creates a script, use a safe read-only script such as: + +```text +scripts/stage1_inventory.py +``` + +Expected command: + +```bash +python scripts/stage1_inventory.py --raw-dir data/raw --out-dir out/stage1 +``` + +Rules: + +- script must not modify `data/raw`; +- script must create only files under `out/stage1` unless otherwise necessary; +- script must log row counts and schema hashes; +- script must tolerate multiple sheets; +- script must not drop unknown values silently; +- script must preserve source traceability. + +--- + +## 18. Verification / Smoke Checks + +Minimum checks: + +```text +- all Excel files discovered; +- raw files unchanged; +- schema groups reproducible; +- output files created; +- row counts logged; +- ambiguous values marked as unknown / needs_confirmation; +- no destructive operations performed. +``` + +If possible, run: + +```bash +python scripts/stage1_inventory.py --raw-dir data/raw --out-dir out/stage1 +``` + +Then report: + +- command run; +- result; +- errors or limitations; +- files generated. + +--- + +## 19. Acceptance Criteria + +The task is accepted if: + +- all required output files are created; +- all 57 Excel files are included in inventory or missing files are explained; +- schema groups are documented; +- column mapping draft is created; +- period, currency, plan/fact detection reports are created; +- DDS and INOUT unique values are extracted; +- `stage1_decisions_needed.md` clearly lists user decisions; +- no raw files are changed; +- no business meaning is silently inferred; +- verification results are reported. + +--- + +## 20. Risks + +| Risk | Impact | Mitigation | +|---|---|---| +| Wrong authoritative files | Stage 1 loads wrong data | Create candidate report, require user approval | +| Period inferred incorrectly | Period-level analytics wrong | Detect candidates, flag conflicts | +| Currency missing or mixed | Amounts not comparable | Preserve raw currency, no conversion in Stage 1 | +| Plan/fact mixed | Incorrect variance analysis | Mark uncertain scenario as `unknown` | +| DDS/INOUT misunderstood | Wrong cash flow classification | Extract values, require mapping approval | +| Raw files modified | Loss of audit trail | Read-only raw handling | +| Broad refactor | New bugs | Minimal diff, out/stage1 only | + +--- + +## 21. Final Codex Prompt + +```markdown +# Task: Prepare Stage 1 Data Contract + +You are working in an existing finance data project. + +## Goal + +Prepare a safe Stage 1 data contract for loading and normalizing Excel files from `data/raw`. + +Do not implement final analytics. +Do not modify raw files. +Do not refactor unrelated ETL. + +## Current known facts + +A previous read-only reconnaissance found 57 Excel files in `data/raw`. + +Schema groups: +- 52 columns: 41 files in `budget_rows`, `dds`, `p-fact` +- 30 columns: 10 files in `dds` +- 38 columns: 5 files in `budget_rows`, `dds` +- 19 columns: 1 file in `cons_budget` + +The main blocker: +Stage 1 cannot be safely designed until authoritative files, period rules, currency rules, plan/fact rules, DDS, INOUT, and target schema are confirmed. + +## Required work + +Create a read-only Stage 1 preparation package. + +## Required outputs + +Create: + +- `out/stage1/file_inventory.csv` +- `out/stage1/schema_groups.md` +- `out/stage1/authoritative_candidates.md` +- `out/stage1/column_mapping_draft.xlsx` +- `out/stage1/column_mapping_draft.md` +- `out/stage1/period_detection_report.md` +- `out/stage1/currency_detection_report.md` +- `out/stage1/plan_fact_detection_report.md` +- `out/stage1/dds_values_report.xlsx` +- `out/stage1/inout_values_report.xlsx` +- `out/stage1/dds_inout_mapping_draft.md` +- `out/stage1/stage1_decisions_needed.md` + +## Rules + +- Do not change files in `data/raw`. +- Do not delete, move, or rename raw files. +- Do not infer business meaning silently. +- If period, currency, plan/fact, DDS, or INOUT are ambiguous, mark as `unknown` or `needs_confirmation`. +- Keep implementation minimal. +- Prefer read-only scripts. +- Log row counts, schema hashes, sheet names, and source file paths. +- Preserve source traceability: every normalized candidate must link back to source file, sheet, and row where possible. + +## Target + +The result should allow the user to approve Stage 1 rules before any real ETL implementation starts. + +## Final response format + +Return: + +1. Summary of generated files. +2. What was discovered. +3. What remains undecided. +4. Risks. +5. Recommended next step. +6. Verification commands run. +``` + +--- + +## 22. Recommended Next Step After This SPEC + +Run Codex with the final prompt from section 21. + +After Codex generates `out/stage1/stage1_decisions_needed.md`, the user should approve or correct the decisions. + +Only after that should Stage 1 ETL implementation begin. diff --git a/docs/migration/generated_artifacts_policy.md b/docs/migration/generated_artifacts_policy.md new file mode 100644 index 0000000..8659194 --- /dev/null +++ b/docs/migration/generated_artifacts_policy.md @@ -0,0 +1,27 @@ +# Generated Artifacts Policy + +Status: Phase 1 migration preparation + +Generated outputs must not be mixed with source code in the public repository by default. + +## Policy + +- `workspace/` is runtime workspace and ignored. +- `publish/` is generated output and ignored by default. +- `publish_assets/` is generated/publication support and ignored by default. +- `release_snapshots/` is release evidence and ignored by default unless selected. +- Final release artifacts can be attached to GitHub Releases later after review. + +## Decision Table + +| Folder | Purpose | Publish default | Risk | Allowed future publication method | +|---|---|---:|---|---| +| `workspace/` | Generated inventory, chunks, clean notes, source cards, knowledge workspace, logs | No | Contains intermediate outputs derived from raw/private inputs | Rebuild locally; do not commit by default | +| `publish/` | Generated user-facing KB and governance reports | No | May contain derived private/raw content or stale release state | Select reviewed artifacts for GitHub Releases or docs only after approval | +| `publish_assets/` | Publication support artifacts | No | May duplicate generated publish content | Review and explicitly approve selected assets | +| `release_snapshots/` | Release evidence snapshots | No | Release-specific evidence may include generated/private-derived content | Attach reviewed snapshot to a release after approval | +| `artifacts/`, `runs/`, `outputs/`, `output/` | Runtime outputs if created later | No | Local/generated artifacts and logs | Keep ignored; publish only reviewed extracts | + +## Current Phase 1 decision + +No generated artifacts are edited, moved, deleted, or approved for publication in Phase 1. diff --git a/docs/migration/github_transfer_plan.md b/docs/migration/github_transfer_plan.md new file mode 100644 index 0000000..0008b7b --- /dev/null +++ b/docs/migration/github_transfer_plan.md @@ -0,0 +1,57 @@ +# GitHub Transfer Plan + +Status: Phase 2 plan only. Do not execute in Phase 1. + +Destination repository: + +```text +https://github.com/sergstack/Build-your-knowledge-base +``` + +The destination repository already has `main`, one initial commit, and `README.md` according to the Phase 0 snapshot. Do not overwrite it blindly. + +## Phase 2 Strategy + +1. Ensure the working tree contains only approved migration-prep changes. +2. Run the final redacted secret scan. +3. Run tests. +4. Add remote only after user approval: + + ```bash + git remote add origin https://github.com/sergstack/Build-your-knowledge-base.git + ``` + +5. Fetch remote: + + ```bash + git fetch origin + ``` + +6. Inspect remote `main`: + + ```bash + git ls-remote --heads origin + git log --oneline --decorate --all -5 + ``` + +7. Create a migration branch: + + ```bash + git switch -c chore/github-migration-foundation + ``` + +8. Commit approved files only. +9. Push the branch: + + ```bash + git push -u origin chore/github-migration-foundation + ``` + +10. Open a pull request into `main`. +11. Do not force-push. +12. Review the PR file list carefully. +13. Merge only after CI passes and user approval. + +## Stop Conditions + +Stop and ask if unrelated histories appear, the remote has changed unexpectedly, a secret scan finds live credentials, raw transcripts are staged, generated outputs are staged, or CI fails for reasons unrelated to migration-prep docs/foundation files. diff --git a/docs/migration/publication_scope.md b/docs/migration/publication_scope.md new file mode 100644 index 0000000..a9dc877 --- /dev/null +++ b/docs/migration/publication_scope.md @@ -0,0 +1,85 @@ +# Publication Scope + +Status: Phase 1 migration preparation + +## Publish by default + +These files and folders are intended for public repository publication after final review: + +```text +README.md +AGENTS.md +CODEX.md +CURRENT_SCOPE.md +DATA_CONTRACTS.md +RUNBOOK.md +requirements.txt +.env.example +.gitignore +src/ +scripts/ +tests/ +prompts/ +docs/ +.github/ +CHANGELOG.md +CONTRIBUTING.md +``` + +## Publish after review + +These files may be useful publicly, but need review for stale scope, private context, or release-specific claims: + +```text +SPEC*.md +plan.md +tasks.md +ACCEPTANCE_CRITERIA.md +MODEL_ROUTING.md +SCOPE_LOCK.md +SKILLS_SEQUENCE.md +DIRTY_WORKTREE_CLASSIFICATION.md +DOCS_CONTRACT_COMMIT_PLAN.md +RAW_INPUT_* docs +WRAPPER_SAFETY_REVIEW.md +selected publication reports +selected release snapshots +``` + +## Do not publish by default + +These files and folders are private, generated, local, or cache-like: + +```text +.env +input/raw/ +workspace/ +publish/ +publish_assets/ +release_snapshots/ +.pytest_cache/ +__pycache__/ +.DS_Store +*.log +*.tmp +*.bak +local credentials +raw transcripts +runtime outputs +generated intermediate outputs +``` + +## Unknown / needs user decision + +These require explicit review before publication: + +```text +public sample raw data +selected final release artifacts +selected compact KB outputs +selected example outputs +``` + +## Current Phase 1 decision + +Phase 1 prepares the repository foundation only. It does not approve publishing raw transcripts, generated outputs, release snapshots, or private local runtime files. diff --git a/docs/migration/raw_input_policy.md b/docs/migration/raw_input_policy.md new file mode 100644 index 0000000..684771f --- /dev/null +++ b/docs/migration/raw_input_policy.md @@ -0,0 +1,34 @@ +# Raw Input Policy + +Status: Phase 1 migration preparation + +## Policy + +`input/raw/` is private by default. + +Raw transcripts must not be published by default. They may contain personal context, operational details, credentials shown in tutorials, private server details, financial or business references, copyrighted/private source material, or other content that is not appropriate for a public repository. + +Only sanitized samples may be published. + +Sanitized samples must remove: + +- credentials; +- personal data; +- private server details; +- API keys and tokens; +- passwords and secret-like examples; +- financial/private operational details; +- copyrighted/private source material that is not approved for redistribution. + +## Recommended Future Structure + +```text +input/ + samples/ + README.md + sample_transcript_sanitized.txt + raw/ + # ignored/private +``` + +No sample data is created in Phase 1 because the current raw inputs were not reviewed as public-safe samples. diff --git a/input/.gitkeep b/input/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/input/README.md b/input/README.md new file mode 100644 index 0000000..8af598d --- /dev/null +++ b/input/README.md @@ -0,0 +1,12 @@ +# Input Data + +Raw/private inputs are intentionally excluded from the public repository. + +Use only sanitized sample files in this directory. + +Do not commit: +- raw transcripts; +- credentials; +- personal data; +- private operational notes; +- generated runtime files. diff --git a/plan.md b/plan.md new file mode 100644 index 0000000..40d9253 --- /dev/null +++ b/plan.md @@ -0,0 +1,84 @@ +# Plan + +## Missing inputs +- None + +## Scope assumptions +- Use `SPEC_synthesis_layer.md` as the task-specific source of truth because root `SPEC.md` describes a different MVP-10 search CLI task. +- Build deterministic synthesis skeletons by default; do not claim high-quality semantic synthesis when evidence is weak. +- Existing compact package files under `publish/chatgpt_project_compact/` must remain in place. +- Source folders are read-only inputs. + +## Affected files / areas +- `scripts/build_synthesis_layer.py` +- `publish/chatgpt_project_compact/KB__05_CANONICAL_CONCEPTS.md` +- `publish/chatgpt_project_compact/KB__06_OPERATIONAL_FRAMEWORKS.md` +- `publish/chatgpt_project_compact/KB__07_PATTERNS_AND_FAILURES.md` +- `publish/chatgpt_project_compact/KB__08_USE_CASES_FOR_SERGEY.md` +- `publish/chatgpt_project_compact/SYNTHESIS_MANIFEST.md` +- Read-only input areas: + - `workspace/source_cards/` + - `workspace/clean_notes/` + - `publish/markdown_kb/` + - `publish/chatgpt_project/` + +## Steps +1. Inspect source availability and file patterns in the allowed input folders. + - Outcome: available source cards, clean notes, published markdown, and compact/navigation files are known. +2. Add `scripts/build_synthesis_layer.py` with CLI parsing for `--max-sources-per-concept`, `--output`, and optional non-default `--use-llm`. + - Outcome: script runs from project root with Python standard library in default mode. +3. Implement safe source discovery and evidence extraction. + - Outcome: sources are read in the required priority order, forbidden folders are ignored, UTF-8 is preserved, and source references are collected. +4. Implement deterministic candidate concept extraction. + - Outcome: headings, filenames, and repeated terms produce a deterministic candidate list while all required concepts are included. +5. Generate `KB__05_CANONICAL_CONCEPTS.md`. + - Outcome: 20-50 concepts exist; each includes definition, why it matters, source-backed facts, operational use, related concepts, anti-patterns, limitations, evidence, and confidence. +6. Generate `KB__06_OPERATIONAL_FRAMEWORKS.md`. + - Outcome: 10-20 frameworks exist; each includes purpose, trigger, inputs, ordered steps, outputs, QA gates, failure modes, Sergey usage, evidence, and confidence. +7. Generate `KB__07_PATTERNS_AND_FAILURES.md`. + - Outcome: required patterns and anti-patterns exist, including `Inventing definitions where source says not found`, with meaning, usage, rationale, risk, evidence, and confidence. +8. Generate `KB__08_USE_CASES_FOR_SERGEY.md`. + - Outcome: all ten required Sergey use cases exist with goal, inputs, steps, output, prompt template, risks, evidence, and confidence. +9. Generate `SYNTHESIS_MANIFEST.md`. + - Outcome: manifest records timestamp, inputs used, outputs created, created entities, evidence quality summary, warnings, and validation status. +10. Add internal validation before script completion. + - Outcome: output files exist, are UTF-8 readable, required sections are present, evidence/confidence fields are present, and forbidden raw source markers are absent. +11. Run required validation commands. + - Outcome: script generation, output listing, file listing, and `py_compile` all complete successfully. + +## Dependencies +- Step 2 depends on Step 1. +- Step 3 depends on Step 2. +- Step 4 depends on Step 3. +- Step 5 depends on Steps 3 and 4. +- Step 6 depends on Step 3. +- Step 7 depends on Step 3. +- Step 8 depends on Step 3. +- Step 9 depends on Steps 5, 6, 7, and 8. +- Step 10 depends on Step 9. +- Step 11 depends on Step 10. + +## Risks +- Deterministic synthesis may produce useful skeletons rather than polished knowledge. +- Some required concepts, frameworks, patterns, or use cases may have weak evidence only. +- Source evidence may be noisy or uneven across topics. +- Optional LLM mode could add complexity and should remain non-default if implemented. +- Copying too much source text could recreate noisy raw material instead of distilled knowledge. + +## Validation strategy +- Run `python3 scripts/build_synthesis_layer.py`. +- Run `ls -lah publish/chatgpt_project_compact`. +- Run `find publish/chatgpt_project_compact -maxdepth 1 -type f | sort`. +- Run `python3 -m py_compile scripts/build_synthesis_layer.py`. +- Confirm all five new synthesis output files exist. +- Confirm outputs are UTF-8 readable. +- Confirm every concept has evidence and confidence. +- Confirm every framework has trigger, inputs, ordered steps, outputs, and QA gates. +- Confirm anti-patterns include `Inventing definitions where source says not found`. +- Confirm raw transcript paths/content are absent from outputs. +- Confirm `SYNTHESIS_MANIFEST.md` includes warnings and evidence quality summary. +- Rerun the script to confirm repeatability. + +## Parallel work +- Steps 5, 6, 7, and 8 can be implemented in parallel after Steps 3 and 4. +- Steps 9, 10, and 11 must run after all output generators are complete. diff --git a/prompts/clean_note_ru.md b/prompts/clean_note_ru.md new file mode 100644 index 0000000..a688743 --- /dev/null +++ b/prompts/clean_note_ru.md @@ -0,0 +1,35 @@ +prompt_version: clean_note_ru_v1 + +Сформируй Clean Note только по предоставленным transcript chunks. + +Формат ответа — строгий контракт: +- Верни только содержимое Clean Note, без вступлений и пояснений. +- Не оборачивай ответ в ```markdown, ``` или любой другой fenced code block. +- Не добавляй текст до первой секции. +- Первая строка ответа должна быть ровно `## Short Summary`. +- Последняя секция ответа должна быть `## Review Notes`. +- Не добавляй текст после секции `## Review Notes`. +- Не меняй названия секций. +- Не добавляй дополнительные top-level секции. +- Не используй YAML/front matter. + +Правила: +- Не добавляй внешние знания. +- Не выдумывай факты. +- Если данных нет, пиши `not found in source`. +- Сохраняй практические процедуры и workflow. +- Добавляй evidence pointers по chunk_id. +- Не включай скрытые рассуждения или `` блоки. +- В `## Evidence Pointers` используй только реальные chunk_id из входа, например `chunk_001`. +- Если тезис не подтверждается конкретным chunk_id, перенеси его в `## Open Questions` или замени на `not found in source`. + +Верни Markdown строго с этими секциями и в этом порядке: +## Short Summary +## Main Ideas +## Procedures / Workflow +## Concepts +## Practical Rules +## Risks / Caveats +## Evidence Pointers +## Open Questions +## Review Notes diff --git a/prompts/kb_build_ru.md b/prompts/kb_build_ru.md new file mode 100644 index 0000000..d0b4f1b --- /dev/null +++ b/prompts/kb_build_ru.md @@ -0,0 +1,11 @@ +prompt_version: kb_build_ru_v1 + +Собери Knowledge Base только по source cards. + +Правила: +- Не добавляй внешние знания. +- Не выдумывай topics, concepts, examples, procedures или risks. +- Каждый важный тезис должен ссылаться на source card или evidence внутри source card. +- Если данных нет, пиши `not found in source`. +- Unsupported или unclear claims отправляй в review queue. +- Не включай скрытые рассуждения или `` блоки. diff --git a/prompts/source_card_ru.md b/prompts/source_card_ru.md new file mode 100644 index 0000000..4926100 --- /dev/null +++ b/prompts/source_card_ru.md @@ -0,0 +1,35 @@ +prompt_version: source_card_ru_v1 + +Сформируй Source Card только по clean note и transcript chunk evidence. + +Формат ответа — строгий контракт: +- Верни только содержимое Source Card, без вступлений и пояснений. +- Не оборачивай ответ в ```markdown, ``` или любой другой fenced code block. +- Не добавляй текст до первой секции. +- Первая строка ответа должна быть ровно `## Core Topic`. +- Последняя секция ответа должна быть `## Human Review`. +- Не добавляй текст после секции `## Human Review`. +- Не меняй названия секций. +- Не добавляй дополнительные top-level секции. +- Не используй YAML/front matter. + +Правила: +- Не добавляй внешние знания. +- Не выдумывай concepts, procedures, examples или risks. +- Каждый важный тезис должен иметь evidence через chunk_id или секцию clean note. +- Если данных нет, пиши `not found in source`. +- Unsupported или unclear claims перечисляй в `## Not Found / Unclear`. +- Не включай скрытые рассуждения или `` блоки. +- В evidence используй только реальные chunk_id из входа, например `chunk_001`. +- Если тезис не подтверждается конкретным chunk_id или clean note, перенеси его в `## Not Found / Unclear` или замени на `not found in source`. + +Верни Markdown строго с этими секциями и в этом порядке: +## Core Topic +## Key Concepts +## Procedures / Workflows +## Practical Rules +## Examples +## Risks / Caveats +## Not Found / Unclear +## Tags +## Human Review diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..40ddeab --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +pytest>=8.0 +PyYAML>=6.0 diff --git a/scripts/build_chatgpt_compact_kb.py b/scripts/build_chatgpt_compact_kb.py new file mode 100644 index 0000000..3b6a947 --- /dev/null +++ b/scripts/build_chatgpt_compact_kb.py @@ -0,0 +1,945 @@ +#!/usr/bin/env python3 +import argparse +import sys +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +CHATGPT_DIR = ROOT / "publish" / "chatgpt_project" +MARKDOWN_KB_DIR = ROOT / "publish" / "markdown_kb" +DEFAULT_OUTPUT = Path("publish/chatgpt_project_compact") + +REQUIRED_OUTPUTS = [ + "KB__00_INDEX.md", + "KB__01_NAVIGATION.md", + "KB__02_CONTENT.md", + "KB__03_WORKFLOWS_TRACEABILITY.md", + "KB__04_SMOKE_QA.md", + "KB__RELEASE_MANIFEST.md", + "KB__CHANGELOG.md", + "KB__REVIEW_QUEUE.md", + "KB__CARD_SCHEMA.md", + "KB__CONFIDENCE_RULES.md", + "KB__PROMOTION_GATES.md", + "KB__RETRIEVAL_QA.md", + "KB__DEDUPLICATION.md", + "KB__USE_CASE_ROUTING.md", + "README.md", + "MANIFEST.md", +] + +GOVERNANCE_OUTPUTS = [ + "KB__RELEASE_MANIFEST.md", + "KB__CHANGELOG.md", + "KB__REVIEW_QUEUE.md", + "KB__CARD_SCHEMA.md", + "KB__CONFIDENCE_RULES.md", + "KB__PROMOTION_GATES.md", + "KB__RETRIEVAL_QA.md", + "KB__DEDUPLICATION.md", + "KB__USE_CASE_ROUTING.md", +] + +MANAGED_PIPELINE = ( + "transcript -> chunk -> source card -> concept / workflow / pattern extraction " + "-> grounded synthesis -> publish package -> compact package -> automated smoke QA " + "-> automated acceptance check -> next scope decision -> use-case routing" +) + +FORBIDDEN_INPUT_PARTS = ( + "workspace/raw", + "workspace/clean_notes", + "workspace/source_cards", + "workspace/chunks", + "/temp/", + "/logs/", +) +FORBIDDEN_LINE_MARKERS = ( + "workspace/raw", + "workspace/clean_notes", + "workspace/source_cards", + "workspace/chunks", + ".source_card.", +) +TEMP_OR_LOG_SUFFIXES = (".tmp", ".temp", ".log") + + +@dataclass +class Manifest: + timestamp: str + included: list[tuple[str, int]] = field(default_factory=list) + skipped: list[tuple[str, int, str]] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + validation: list[tuple[str, str]] = field(default_factory=list) + sanitized_lines: int = 0 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build compact ChatGPT Project KB package.") + parser.add_argument( + "--max-file-mb", + type=float, + default=1.0, + help="Maximum markdown input file size in MB for content merge.", + ) + parser.add_argument( + "--output", + default=str(DEFAULT_OUTPUT), + help="Output directory for compact package.", + ) + return parser.parse_args() + + +def rel(path: Path) -> str: + try: + return path.resolve().relative_to(ROOT).as_posix() + except ValueError: + return path.as_posix() + + +def progress(message: str) -> None: + print(f"Progress: {message}") + + +def fail(message: str) -> None: + print(f"error: {message}", file=sys.stderr) + raise SystemExit(1) + + +def require_file(path: Path) -> None: + if not path.is_file(): + fail(f"required source file is missing: {rel(path)}") + + +def require_dir(path: Path) -> None: + if not path.is_dir(): + fail(f"required source folder is missing: {rel(path)}") + + +def read_text(path: Path) -> str: + try: + return path.read_text(encoding="utf-8") + except UnicodeDecodeError as exc: + raise ValueError(f"not UTF-8 readable: {rel(path)}") from exc + + +def write_text(path: Path, content: str, manifest: Manifest) -> None: + path.write_text(normalize_blank_lines(content).rstrip() + "\n", encoding="utf-8") + manifest.included.append((rel(path), path.stat().st_size)) + + +def normalize_blank_lines(text: str) -> str: + lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n") + output: list[str] = [] + blank = False + for line in lines: + if line.strip(): + output.append(line.rstrip()) + blank = False + elif not blank: + output.append("") + blank = True + return "\n".join(output) + + +def is_binary(path: Path) -> bool: + try: + chunk = path.read_bytes()[:4096] + except OSError: + return True + return b"\x00" in chunk + + +def should_skip_input(path: Path, max_bytes: int) -> str | None: + name = path.name + lower = name.lower() + path_text = rel(path) + if any(part.startswith(".") for part in path.relative_to(ROOT).parts): + return "hidden file" + if any(part in path_text for part in FORBIDDEN_INPUT_PARTS): + return "forbidden source path" + if lower.endswith(".txt"): + return ".txt file" + if lower.endswith(TEMP_OR_LOG_SUFFIXES): + return "temp or log file" + if path.suffix.lower() != ".md": + return "not a markdown file" + size = path.stat().st_size + if size > max_bytes: + return f"larger than max-file-mb ({size} bytes > {max_bytes} bytes)" + if is_binary(path): + return "binary file" + return None + + +def sanitize_content(text: str, manifest: Manifest) -> str: + kept: list[str] = [] + removed = 0 + for line in text.splitlines(): + if any(marker in line for marker in FORBIDDEN_LINE_MARKERS): + removed += 1 + continue + kept.append(line) + if removed: + manifest.sanitized_lines += removed + return "\n".join(kept) + + +def source_block(path: Path, manifest: Manifest, *, sanitize: bool = True) -> str: + text = read_text(path) + if sanitize: + text = sanitize_content(text, manifest) + return f"\n\n{text.strip()}\n" + + +def discover_markdown_kb(max_bytes: int, manifest: Manifest) -> list[Path]: + files: list[Path] = [] + for path in sorted(MARKDOWN_KB_DIR.rglob("*"), key=lambda item: rel(item).lower()): + if not path.is_file(): + continue + reason = should_skip_input(path, max_bytes) + if reason: + manifest.skipped.append((rel(path), path.stat().st_size, reason)) + continue + files.append(path) + return files + + +def extract_section(text: str, heading: str) -> str | None: + lines = text.splitlines() + start = None + for idx, line in enumerate(lines): + if line.strip().lower() == f"## {heading}".lower(): + start = idx + break + if start is None: + return None + end = len(lines) + for idx in range(start + 1, len(lines)): + line = lines[idx] + if line.startswith("## ") and not line.startswith("### "): + end = idx + break + return "\n".join(lines[start:end]).strip() + + +def extract_ai_context_sections(path: Path, manifest: Manifest) -> str: + text = read_text(path) + sections: list[str] = [] + missing: list[str] = [] + for heading in ("Publish Metadata", "Knowledge Index", "Source Traceability"): + section = extract_section(text, heading) + if section is None: + missing.append(heading) + else: + sections.append(section) + + if missing: + manifest.warnings.append( + "AI context section extraction fallback used; missing headings: " + + ", ".join(missing) + ) + sections = ["\n".join(text.splitlines()[:300])] + + cleaned = sanitize_content("\n\n".join(sections), manifest) + if "workspace/source_cards" in "\n\n".join(sections): + manifest.warnings.append( + "Removed source-card path lines from AI context excerpts to keep compact package clean." + ) + return cleaned + + +def build_index(timestamp: str, output_dir: Path) -> str: + files = "\n".join(f"- `{name}`" for name in REQUIRED_OUTPUTS) + return f"""# Compact ChatGPT Project KB Package + +## Package purpose +This package condenses the published Knowledge Base into a governed markdown package for upload into ChatGPT Project. + +## Generated timestamp +{timestamp} + +## Canonical managed pipeline +{MANAGED_PIPELINE} + +## Source folders used +- `publish/chatgpt_project/` +- `publish/markdown_kb/` + +## Output files +{files} + +## Recommended usage in ChatGPT Project +Upload all files from `{rel(output_dir)}` into the same ChatGPT Project knowledge area. Start with `KB__00_INDEX.md`, then use `KB__01_NAVIGATION.md` to orient questions and `KB__04_SMOKE_QA.md` to test retrieval. + +## Governance rule +Smoke QA is not the final stage. Treat the package as not production-promoted unless `KB__PROMOTION_GATES.md`, `KB__RETRIEVAL_QA.md`, and `KB__RELEASE_MANIFEST.md` show deterministic pass conditions. + +## Limitations +- Compact files are derived from published markdown only. +- Raw transcripts, source cards, clean notes, chunks, temp files, and logs are excluded. +- Large markdown files may be skipped according to `--max-file-mb`. +- This package does not add semantic search, embeddings, vector databases, web UI, or autonomous retrieval. + +## Update procedure +Regenerate the publish outputs first if needed, then run: + +```bash +python3 scripts/build_chatgpt_compact_kb.py +``` +""" + + +def build_navigation(manifest: Manifest) -> str: + managed = f"""# Navigation + +## Managed Knowledge System +Canonical pipeline: + +```text +{MANAGED_PIPELINE} +``` + +Smoke QA is a pre-acceptance gate. Final use requires automated acceptance, next scope decision, and use-case routing. + +## Governed compact files +- `KB__00_INDEX.md` — package entrypoint. +- `KB__01_NAVIGATION.md` — navigation and routing. +- `KB__02_CONTENT.md` — compact published content. +- `KB__03_WORKFLOWS_TRACEABILITY.md` — workflows, traceability, gates, and promotion control. +- `KB__04_SMOKE_QA.md` — automated smoke and retrieval QA requirements. +- `KB__05_CANONICAL_CONCEPTS.md` — generated by synthesis layer. +- `KB__06_OPERATIONAL_FRAMEWORKS.md` — generated by synthesis layer. +- `KB__07_PATTERNS_AND_FAILURES.md` — generated by synthesis layer. +- `KB__08_USE_CASES_FOR_SERGEY.md` — generated by synthesis layer. +- `KB__RELEASE_MANIFEST.md` — release governance and promotion state. +- `KB__REVIEW_QUEUE.md` — weak, unsupported, duplicate, conflict, and traceability items. +- `KB__CARD_SCHEMA.md` — card passport and review status schema. +- `KB__CONFIDENCE_RULES.md` — confidence and promotion rules. +- `KB__PROMOTION_GATES.md` — automated gates before production promotion. +- `KB__RETRIEVAL_QA.md` — deterministic retrieval QA schema and checks. +- `KB__DEDUPLICATION.md` — duplicate detection and merge policy. +- `KB__USE_CASE_ROUTING.md` — use-case routing contract. + +## User aliases +- `7 великих на GitHub` routes to these source cards: Hamel Husain (`ba8f38829c4c`), Simon Willison (`047131d56e3f`), Sebastian Raschka (`0289a622670f`), Paul Gauthier (`1b55b74e10a7`), Jason Liu (`6d3f1535b610`), Georgi Gerganov (`fbca6860f502`), Phil Wang (`3aab3b710e61`). +- When a prompt mentions `7 великих на GitHub`, answer from this seven-source subset first, then state if additional KB sources were used. +""" + parts = [managed] + for name in ("INDEX.md", "CONCEPT_MAP.md", "KB_USAGE_GUIDE.md"): + parts.append(source_block(CHATGPT_DIR / name, manifest)) + return "\n\n".join(parts) + + +def build_content(markdown_files: list[Path], manifest: Manifest) -> str: + parts = ["# Content\n"] + for path in markdown_files: + parts.append(f"\n\n{sanitize_content(read_text(path), manifest).strip()}\n") + return "\n\n".join(parts) + + +def build_workflows_traceability(manifest: Manifest) -> str: + managed = f"""# Workflows and Traceability + +## Canonical managed pipeline +```text +{MANAGED_PIPELINE} +``` + +## Traceability chain +Every canonical concept must remain traceable through: + +```text +concept -> workflow -> evidence -> source card -> transcript/chunk +``` + +## Acceptance / Promotion Gates +1. Publish gate — publish package is built successfully. +2. Boundary gate — publish package excludes `.txt`, raw chunks, temp files, logs, and source-card paths. +3. Preservation gate — helper/navigation files are preserved. +4. Consumer QA gate — smoke questions are answered with grounded answers. +5. Acceptance gate — pass/fail, residual risks, known gaps, and next scope are recorded. +6. Promotion gate — embeddings, semantic search, vector DB, web UI, and agentic automation remain blocked until acceptance passes. + +## Promotion control +- Any `unsupported` item blocks promotion. +- `weak` items are allowed only when listed in `KB__REVIEW_QUEUE.md`. +- Weak or unsupported items must not be promoted as canonical facts or operational framework evidence. +""" + parts = [managed] + for name in ("WORKFLOW_MAP.md", "TRACEABILITY_GUIDE.md"): + parts.append(source_block(CHATGPT_DIR / name, manifest)) + ai_path = CHATGPT_DIR / "AI_KB_Context_File_v1.0.md" + parts.append(f"\n\n{extract_ai_context_sections(ai_path, manifest)}\n") + return "\n\n".join(parts) + + +def build_smoke_qa(manifest: Manifest) -> str: + source = source_block(CHATGPT_DIR / "SMOKE_QUESTIONS.md", manifest) + return f"""# Smoke QA + +{source} + +## Automated smoke test sequence +1. Upload all compact package files into ChatGPT Project. +2. Ask the first smoke question from the source list. +3. Confirm the answer cites or names the retrieved compact KB section. +4. Repeat with one navigation question, one workflow question, and one traceability question. +5. Record pass or fail for each answer in deterministic QA output before acceptance. +6. Do not treat smoke QA as final acceptance. + +## Pass/fail checklist +- [ ] Answer is grounded in uploaded compact KB files. +- [ ] Answer identifies the relevant compact file or section. +- [ ] Answer uses confidence labels: strong, medium, weak, or unsupported. +- [ ] Answer preserves traceability to evidence. +- [ ] Answer lists unsupported claims instead of promoting them. +- [ ] Answer does not cite raw transcripts, source cards, clean notes, chunks, temp files, or logs. +- [ ] Answer separates facts from assumptions when the KB is incomplete. +- [ ] Answer reports uncertainty instead of inventing missing details. + +## Automated acceptance checklist +- acceptance_status: pass/fail +- residual_risks: required +- blocked_promotions: required +- unresolved_reviews: required +- next_scope: required +- production_ready: yes/no + +## Retrieval QA output schema +- question +- expected_source +- actual_source +- retrieval_status +- grounding_status +- confidence_status +- unsupported_claims +- final_verdict + +## Reporting retrieved chunks back to ChatGPT +When checking an answer, paste the compact file name, section heading, and the retrieved excerpt. Use this format: + +```text +file: KB__02_CONTENT.md +section: +retrieved excerpt: +question: +issue: +``` +""" + + +def build_release_manifest(timestamp: str, output_dir: Path, manifest: Manifest) -> str: + source_inputs = "\n".join( + f"- `{path}`" for path in ("publish/chatgpt_project/", "publish/markdown_kb/") + ) + compact_outputs = "\n".join(f"- `{name}`" for name in REQUIRED_OUTPUTS) + return f"""# KB Release Manifest + +## kb_version +managed-knowledge-system-v1 + +## build_date +{timestamp} + +## source_inputs +{source_inputs} + +## processed_transcripts +- Determined upstream by inventory/source-card pipeline. +- This compact release does not read raw transcript files. + +## generated_cards +- Source Card +- Concept Card +- Workflow Card +- Pattern Card +- QA Card +- Navigation Card +- Use Case Card + +## publish_outputs +- `publish/chatgpt_project/` +- `publish/markdown_kb/` + +## compact_package_outputs +{compact_outputs} + +## smoke_qa_status +pending_automated_check + +## acceptance_status +blocked_until_automated_gates_pass + +## residual_risks +- Synthesis files are generated after the compact base package. +- Weak and unsupported evidence must remain visible in `KB__REVIEW_QUEUE.md`. + +## blocked_items +- embeddings +- semantic search +- vector DB +- web UI +- agentic workflows +- autonomous retrieval + +## next_scope +- Run automated retrieval QA and acceptance checks after package generation. + +## promoted_to_production +no + +## release_path +`{rel(output_dir)}` +""" + + +def build_changelog(timestamp: str) -> str: + return f"""# KB Changelog + +## Version +managed-knowledge-system-v1 + +## Date +{timestamp} + +## Added +- Release manifest layer. +- Card passport schema. +- Confidence rules. +- Promotion gates. +- Retrieval QA layer. +- Deduplication rules. +- Automated review queue contract. +- Use-case routing contract. + +## Updated +- Canonical pipeline now ends after automated acceptance, next scope decision, and use-case routing. +- Smoke QA is a pre-acceptance gate, not final readiness. + +## Deprecated +- Treating compact package export as production readiness. + +## Removed +- None. + +## Residual Risks +- Weak or unsupported synthesis must remain blocked from production promotion. + +## Open Review Items +- See `KB__REVIEW_QUEUE.md`. +""" + + +def build_review_queue() -> str: + return """# KB Review Queue + +## Purpose +Automated queue for weak cards, unsupported claims, duplicate candidates, conflicting concepts, unresolved workflows, missing traceability, stale concepts, and low-confidence synthesis. + +## Queue status +- review_required: yes +- reason: acceptance is blocked until automated synthesis and retrieval QA refresh this queue. + +## Required item classes +- weak cards +- unsupported claims +- duplicate candidates +- conflicting concepts +- unresolved workflows +- missing traceability +- stale concepts +- low-confidence synthesis + +## Promotion rule +Items listed here are not production-approved. Unsupported items block promotion. Weak items may remain only if they are not promoted as canonical fact or operational framework evidence. +""" + + +def build_card_schema() -> str: + fields = "\n".join( + f"- {name}" + for name in ( + "card_id", + "card_type", + "source_id", + "related_source_ids", + "title", + "summary", + "key_concepts", + "workflows", + "patterns", + "anti_patterns", + "risks", + "evidence", + "confidence", + "review_status", + "related_use_cases", + "created_at", + "updated_at", + ) + ) + return f"""# Card Passport + +## Mandatory fields +{fields} + +## Required card types +- Source Card +- Concept Card +- Workflow Card +- Pattern Card +- QA Card +- Navigation Card +- Use Case Card + +## Required review_status values +- approved +- review_required +- weak +- unsupported +- deprecated +- duplicate_candidate + +## Rule +Cards without a complete passport are not production-approved and must be listed in `KB__REVIEW_QUEUE.md`. +""" + + +def build_confidence_rules() -> str: + return """# Confidence Rules + +## Confidence Levels +- strong +- medium +- weak +- unsupported + +## strong +Confirmed by source cards, canonical KB, and multiple grounded references. + +## medium +Confirmed by one package file or limited evidence. + +## weak +Interpretation, synthesis, or recommendation. + +## unsupported +Not found in the KB. + +## Promotion Rules +Weak and unsupported items: +- cannot be promoted to canonical concepts; +- cannot be included in operational frameworks as grounded fact; +- cannot be used as grounded fact; +- require automated review queue tracking. + +Unsupported items block promotion. Weak items are allowed only when listed in `KB__REVIEW_QUEUE.md` and excluded from production-approved claims. +""" + + +def build_promotion_gates() -> str: + return """# Acceptance / Promotion Gates + +## Publish gate +- publish package built successfully. + +## Boundary gate +- no `.txt` files; +- no raw chunks; +- no temp files; +- no logs; +- no source-card paths; +- no unresolved duplicate drafts. + +## Preservation gate +- helper/navigation files are preserved. + +## Consumer QA gate +- smoke questions pass with grounded answers. + +## Acceptance gate +- pass/fail recorded; +- residual risks recorded; +- known gaps recorded; +- next scope recorded. + +## Promotion gate +Only after acceptance gate may the system proceed to: +- embeddings; +- semantic search; +- vector DB; +- web UI; +- agentic workflows; +- autonomous retrieval. + +## Automated blockers +- `unsupported` item present outside review queue; +- weak item promoted as canonical fact; +- retrieval QA unstable; +- boundary gate failure. +""" + + +def build_retrieval_qa() -> str: + return """# Retrieval QA + +## Purpose +Automated retrieval QA verifies local generated files before the package is treated as accepted. + +## Checks +1. Expected file exists. +2. Expected section exists. +3. Traceability text exists. +4. Grounded evidence exists. +5. Unsupported claims are absent or listed. +6. Confidence labels exist. +7. Hallucinated synthesis is blocked. +8. Weak/unsupported content is not promoted as canonical or operational fact. + +## Retrieval QA Output +- question +- expected_source +- actual_source +- retrieval_status +- grounding_status +- confidence_status +- unsupported_claims +- final_verdict + +## Deterministic verdict rules +- `pass`: expected source, section, traceability, evidence, and confidence labels are present. +- `fail`: any required source or section is missing. +- `blocked`: unsupported or weak material is promoted as production-approved. +""" + + +def build_deduplication() -> str: + return """# Deduplication + +## Deduplication Rules +Before publish, automated checks must look for: +- duplicate source cards; +- repeated concepts; +- overlapping workflows; +- multiple names for the same idea; +- stale versions; +- obsolete cards; +- conflicting canonical definitions. + +## Merge Policy +When duplicates are found: +1. Preserve strongest evidence. +2. Preserve traceability. +3. Merge aliases. +4. Mark deprecated versions. +5. Keep canonical card_id stable. + +## Safety rule +Duplicates are not silently deleted. Deprecated and duplicate candidates are listed in `KB__REVIEW_QUEUE.md`. +""" + + +def build_use_case_routing() -> str: + return """# Use Case Layer + +## Required routes +Every strong concept, workflow, or pattern must map to at least one operational route: +- analytical memo +- FP&A +- counterparty audit +- Power BI +- Codex tasks +- AI OS architecture +- model routing +- QA / review +- governance +- automation + +## Use Case Fields +- use_case_id +- applicable_roles +- required_confidence +- required_sources +- workflows_used +- operational_risks +- recommended_models + +## Routing rule +Use cases require `medium` or `strong` confidence for operational recommendations. Weak use cases remain in review queue. Unsupported use cases block promotion. + +## User aliases +- `7 великих на GitHub` = Hamel Husain (`ba8f38829c4c`), Simon Willison (`047131d56e3f`), Sebastian Raschka (`0289a622670f`), Paul Gauthier (`1b55b74e10a7`), Jason Liu (`6d3f1535b610`), Georgi Gerganov (`fbca6860f502`), Phil Wang (`3aab3b710e61`). +- Route this alias to the seven source cards above before using the wider KB. +""" + + +def build_readme(output_dir: Path) -> str: + return f"""# Compact ChatGPT Project KB + +This folder contains a governed compact markdown package built from the published Knowledge Base output. + +## Upload into ChatGPT Project +Upload every file from `{rel(output_dir)}` into the same ChatGPT Project knowledge area: + +{chr(10).join(f"- `{name}`" for name in REQUIRED_OUTPUTS)} + +Use `KB__00_INDEX.md` as the starting point and `KB__04_SMOKE_QA.md` to verify retrieval quality. Final readiness requires automated gates in `KB__PROMOTION_GATES.md` and `KB__RELEASE_MANIFEST.md`. + +## Use with Open WebUI +Use Open WebUI as the working chat surface and keep this compact package as the ChatGPT Project reference package. When Open WebUI returns or retrieves a relevant chunk, compare it against the compact file and section names in this package before treating it as grounded. + +## What not to upload +Do not upload raw transcript text files, clean notes, source cards, chunks, temp files, logs, embeddings, vector database files, or unrelated workspace folders. + +## Update command +From the project root, run: + +```bash +python3 scripts/build_chatgpt_compact_kb.py +``` + +Optional: + +```bash +python3 scripts/build_chatgpt_compact_kb.py --max-file-mb 1 --output publish/chatgpt_project_compact +``` +""" + + +def build_manifest(manifest: Manifest) -> str: + included = "\n".join( + f"- `{path}` — {size} bytes" for path, size in sorted(manifest.included) + ) or "- None" + skipped = "\n".join( + f"- `{path}` — {size} bytes — {reason}" + for path, size, reason in sorted(manifest.skipped) + ) or "- None" + warnings = list(manifest.warnings) + if manifest.sanitized_lines: + warnings.append(f"Removed {manifest.sanitized_lines} forbidden source/path lines from compact content.") + warnings_text = "\n".join(f"- {item}" for item in warnings) or "- None" + validation = "\n".join(f"- {name}: {status}" for name, status in manifest.validation) or "- Not run" + return f"""# Manifest + +## Generation timestamp +{manifest.timestamp} + +## Included files +{included} + +## Skipped files +{skipped} + +## Warnings +{warnings_text} + +## Validation status +{validation} +""" + + +def prepare_output_dir(output_dir: Path) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + required = set(REQUIRED_OUTPUTS) + for path in output_dir.iterdir(): + if path.is_file() and path.name not in required: + path.unlink() + + +def validate_output(output_dir: Path, manifest: Manifest) -> None: + required_paths = [output_dir / name for name in REQUIRED_OUTPUTS] + extra_files = sorted(path.name for path in output_dir.iterdir() if path.is_file() and path.name not in REQUIRED_OUTPUTS) + checks: list[tuple[str, bool]] = [] + checks.append(("output folder exists", output_dir.is_dir())) + checks.append(("exact required file count", len([path for path in output_dir.iterdir() if path.is_file()]) == len(REQUIRED_OUTPUTS) and not extra_files)) + checks.append(("all required files exist", all(path.is_file() for path in required_paths))) + + utf8_ok = True + forbidden_ok = True + for path in required_paths: + try: + text = read_text(path) + except ValueError: + utf8_ok = False + text = "" + if any(marker in text for marker in FORBIDDEN_LINE_MARKERS): + forbidden_ok = False + if any(folder in text for folder in ("workspace/raw", "workspace/clean_notes", "workspace/chunks")): + forbidden_ok = False + checks.append(("compact files are UTF-8 readable", utf8_ok)) + checks.append(("forbidden source paths excluded", forbidden_ok)) + checks.append(("manifest lists included files", bool(manifest.included))) + checks.append(("manifest lists skipped files", bool(manifest.skipped))) + combined = "\n".join(read_text(path) for path in required_paths if path.is_file()) + checks.append(("managed pipeline present", MANAGED_PIPELINE in combined)) + checks.append(("promotion gates present", "Acceptance / Promotion Gates" in combined)) + checks.append(("retrieval qa schema present", "final_verdict" in combined)) + checks.append(("release manifest fields present", all(marker in combined for marker in ("kb_version", "acceptance_status", "promoted_to_production")))) + + for name, ok in checks: + manifest.validation.append((name, "pass" if ok else "fail")) + failed = [name for name, ok in checks if not ok] + if failed: + fail("validation failed: " + ", ".join(failed)) + + +def main() -> int: + args = parse_args() + max_bytes = int(args.max_file_mb * 1024 * 1024) + output_dir = (ROOT / args.output).resolve() if not Path(args.output).is_absolute() else Path(args.output) + timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds") + manifest = Manifest(timestamp=timestamp) + + progress("checking source folders") + require_dir(CHATGPT_DIR) + require_dir(MARKDOWN_KB_DIR) + for name in ( + "INDEX.md", + "CONCEPT_MAP.md", + "KB_USAGE_GUIDE.md", + "WORKFLOW_MAP.md", + "TRACEABILITY_GUIDE.md", + "AI_KB_Context_File_v1.0.md", + "SMOKE_QUESTIONS.md", + ): + require_file(CHATGPT_DIR / name) + + progress("preparing output folder") + prepare_output_dir(output_dir) + + progress("discovering markdown KB inputs") + markdown_files = discover_markdown_kb(max_bytes, manifest) + if not markdown_files: + manifest.warnings.append("No markdown KB files were included in KB__02_CONTENT.md.") + + progress("writing compact package files") + write_text(output_dir / "KB__00_INDEX.md", build_index(timestamp, output_dir), manifest) + write_text(output_dir / "KB__01_NAVIGATION.md", build_navigation(manifest), manifest) + write_text(output_dir / "KB__02_CONTENT.md", build_content(markdown_files, manifest), manifest) + write_text(output_dir / "KB__03_WORKFLOWS_TRACEABILITY.md", build_workflows_traceability(manifest), manifest) + write_text(output_dir / "KB__04_SMOKE_QA.md", build_smoke_qa(manifest), manifest) + write_text(output_dir / "KB__RELEASE_MANIFEST.md", build_release_manifest(timestamp, output_dir, manifest), manifest) + write_text(output_dir / "KB__CHANGELOG.md", build_changelog(timestamp), manifest) + write_text(output_dir / "KB__REVIEW_QUEUE.md", build_review_queue(), manifest) + write_text(output_dir / "KB__CARD_SCHEMA.md", build_card_schema(), manifest) + write_text(output_dir / "KB__CONFIDENCE_RULES.md", build_confidence_rules(), manifest) + write_text(output_dir / "KB__PROMOTION_GATES.md", build_promotion_gates(), manifest) + write_text(output_dir / "KB__RETRIEVAL_QA.md", build_retrieval_qa(), manifest) + write_text(output_dir / "KB__DEDUPLICATION.md", build_deduplication(), manifest) + write_text(output_dir / "KB__USE_CASE_ROUTING.md", build_use_case_routing(), manifest) + write_text(output_dir / "README.md", build_readme(output_dir), manifest) + + progress("validating compact package") + write_text(output_dir / "MANIFEST.md", build_manifest(manifest), manifest) + validate_output(output_dir, manifest) + (output_dir / "MANIFEST.md").write_text(normalize_blank_lines(build_manifest(manifest)).rstrip() + "\n", encoding="utf-8") + + progress(f"done: {rel(output_dir)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/build_release_manifest.py b/scripts/build_release_manifest.py new file mode 100644 index 0000000..b9fcd3e --- /dev/null +++ b/scripts/build_release_manifest.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 +import sys +import subprocess +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + +from notes_to_kb.governance import ( # noqa: E402 + ACCEPTANCE_REPORT, + CARD_VALIDATION_REPORT, + COMPACT_DIR, + DEDUPLICATION_REPORT, + GOVERNANCE_STATE, + PROMOTION_REPORT, + RELEASE_MANIFEST_JSON, + RELEASE_AUDIT_SNAPSHOT, + REVIEW_QUEUE_JSON, + RETRIEVAL_QA_RESULTS, + WEAK_EVIDENCE_BACKLOG, + count_by, + read_json, + utc_now, + write_json, +) + + +COMMAND_HISTORY = [ + "python3 scripts/run_publish.py --mode all", + "python3 scripts/build_chatgpt_compact_kb.py", + "python3 scripts/build_synthesis_layer.py", + "python3 scripts/validate_card_passports.py", + "python3 scripts/run_deduplication.py", + "python3 scripts/run_retrieval_qa.py", + "python3 scripts/run_acceptance_gate.py", + "python3 scripts/run_promotion_gate.py", + "python3 scripts/build_release_manifest.py", +] + + +def git_status() -> dict: + result = subprocess.run( + ["git", "status", "--short"], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + return {"status": "not_available", "reason": result.stderr.strip() or "git status failed", "entries": []} + entries = [line for line in result.stdout.splitlines() if line.strip()] + return {"status": "clean" if not entries else "dirty", "entries": entries} + + +def build_weak_evidence_backlog(cards: list[dict], review_queue: dict) -> dict: + weak_cards = [card for card in cards if card.get("confidence") == "weak"] + weak_items = [item for item in review_queue.get("items", []) if item.get("reason") == "weak"] + item_by_card = {item.get("object_id"): item for item in weak_items} + count_by_source = {card["card_id"]: 1 for card in weak_cards} + topic_counts = {} + reason_counts = {} + for card in weak_cards: + topic = card.get("source_title") or card.get("title") or "unknown" + topic_counts[topic] = topic_counts.get(topic, 0) + 1 + reason = item_by_card.get(card.get("card_id"), {}).get("reason", "weak") + reason_counts[reason] = reason_counts.get(reason, 0) + 1 + top_cards = sorted( + weak_cards, + key=lambda card: (card.get("evidence_count", 0), card.get("title", ""), card.get("card_id", "")), + )[:20] + return { + "generated_at": utc_now(), + "command": "python3 scripts/build_release_manifest.py", + "mode": "runtime_governance", + "input_paths": ["publish/card_validation_report.json", "publish/review_queue.json"], + "weak_count": len(weak_cards), + "count_by_source_card": count_by_source, + "count_by_topic": dict(sorted(topic_counts.items(), key=lambda item: (-item[1], item[0]))), + "count_by_reason": dict(sorted(reason_counts.items())), + "top_20_weakest_cards": [ + { + "card_id": card.get("card_id"), + "source_id": card.get("source_id"), + "title": card.get("title"), + "evidence_count": card.get("evidence_count", 0), + "source_file": card.get("source_file"), + "suggested_next_action": "add stronger source evidence or keep as weak non-blocking backlog item", + } + for card in top_cards + ], + "suggested_next_action": "Prioritize weak cards with one evidence reference before enabling downstream semantic products.", + "status": "visible" if weak_cards else "empty", + } + + +def build_release_audit_snapshot(manifest: dict, retrieval_report: dict, card_report: dict, review_queue: dict) -> dict: + return { + "timestamp": utc_now(), + "git_status": git_status(), + "command_history": COMMAND_HISTORY, + "final_metrics": { + "schema_invalid_count": card_report.get("schema_invalid_count", 0), + "unsupported_count": manifest.get("unsupported_count", 0), + "weak_count": manifest.get("weak_count", 0), + "high_severity_review_items": review_queue.get("high_severity_count", 0), + "retrieval_qa_failed": manifest.get("retrieval_qa_failed", 0), + "duplicate_conflicts": manifest.get("duplicate_conflicts", 0), + }, + "route_status": { + "ollama": retrieval_report.get("ollama", {}), + "gemini": retrieval_report.get("gemini", {}), + }, + "production_ready": manifest.get("production_ready", False), + "acceptance_status": manifest.get("acceptance_status"), + "promotion_status": manifest.get("promotion_status"), + } + + +def render_release_markdown(manifest: dict) -> str: + promoted = "yes" if manifest["production_ready"] else "no" + return f"""# KB Release Manifest + +## kb_version +{manifest["kb_version"]} + +## build_date +{manifest["build_date"]} + +## acceptance_status +{manifest["acceptance_status"]} + +## promotion_status +{manifest["promotion_status"]} + +## source_inputs +- source_count: {manifest["source_count"]} + +## processed_transcripts +- controlled by deterministic publish pipeline before runtime governance checks. + +## generated_cards +- card_count: {manifest["card_count"]} + +## publish_outputs +- `publish/` + +## compact_package_outputs +- `publish/chatgpt_project_compact/` + +## smoke_qa_status +- retrieval_qa_passed: {manifest["retrieval_qa_passed"]} +- retrieval_qa_failed: {manifest["retrieval_qa_failed"]} + +## residual_risks +- weak_count: {manifest["weak_count"]} +- unsupported_count: {manifest["unsupported_count"]} +- duplicate_conflicts: {manifest["duplicate_conflicts"]} + +## Weak evidence meaning +`weak_count` means the card has deterministic evidence, but not enough evidence density for `medium` or `strong` confidence. Weak cards remain visible in `publish/weak_evidence_backlog.json` and `KB__REVIEW_QUEUE.md`; they are not production blockers unless they become unsupported or production-facing claims. + +## Optional route status +Gemini skipped is not a blocker when `GEMINI_API_KEY` is absent because deterministic governance is authoritative for this release. Ollama may be available, but it is only an assisted route and is not authority for acceptance or promotion. + +## blocked_items +- review_required_count: {manifest["review_required_count"]} + +## next_scope +- Resolve review queue items before enabling embeddings, semantic search, vector DB, web UI, or agents. + +## promoted_to_production +{promoted} + +## production_ready +{str(manifest["production_ready"]).lower()} + +## Counts +- source_count: {manifest["source_count"]} +- card_count: {manifest["card_count"]} +- weak_count: {manifest["weak_count"]} +- unsupported_count: {manifest["unsupported_count"]} +- retrieval_qa_passed: {manifest["retrieval_qa_passed"]} +- retrieval_qa_failed: {manifest["retrieval_qa_failed"]} +- review_required_count: {manifest["review_required_count"]} + +## Control artifacts +- `publish/governance_state.json` +- `publish/card_validation_report.json` +- `publish/deduplication_report.json` +- `publish/review_queue.json` +- `publish/retrieval_qa_results.json` +- `publish/acceptance_report.json` +- `publish/promotion_report.json` +- `publish/release_manifest.json` +""" + + +def render_review_queue_markdown(review_queue: dict) -> str: + lines = [ + "# KB Review Queue", + "", + "## Summary", + f"- item_count: {review_queue.get('item_count', 0)}", + f"- high_severity_count: {review_queue.get('high_severity_count', 0)}", + "", + "## Promotion impact", + "- Unsupported items block promotion.", + "- Weak items require review queue tracking before operational use.", + "", + "## Items", + ] + for item in review_queue.get("items", []): + lines.extend( + [ + f"### {item['item_id']}", + f"- object_type: {item['object_type']}", + f"- object_id: {item['object_id']}", + f"- reason: {item['reason']}", + f"- severity: {item['severity']}", + f"- recommended_action: {item['recommended_action']}", + f"- source_file: {item['source_file']}", + f"- evidence: {'; '.join(item.get('evidence', []))}", + "", + ] + ) + return "\n".join(lines).rstrip() + "\n" + + +def main() -> int: + card_report = read_json(CARD_VALIDATION_REPORT, {"cards": [], "confidence_counts": {}}) + dedupe_report = read_json(DEDUPLICATION_REPORT, {}) + retrieval_report = read_json(RETRIEVAL_QA_RESULTS, {}) + review_queue = read_json(REVIEW_QUEUE_JSON, {"items": [], "item_count": 0, "high_severity_count": 0}) + acceptance = read_json(ACCEPTANCE_REPORT, {"acceptance_status": "fail"}) + promotion = read_json(PROMOTION_REPORT, {"promotion_status": "blocked", "production_ready": False}) + cards = card_report.get("cards", []) + confidence_counts = card_report.get("confidence_counts", {}) + review_status_counts = count_by(cards, "review_status") + + manifest = { + "kb_version": "managed-knowledge-system-v1", + "build_date": utc_now(), + "command": "python3 scripts/build_release_manifest.py", + "mode": "runtime_governance", + "input_paths": [ + "publish/card_validation_report.json", + "publish/deduplication_report.json", + "publish/retrieval_qa_results.json", + "publish/review_queue.json", + "publish/acceptance_report.json", + "publish/promotion_report.json", + ], + "acceptance_status": acceptance.get("acceptance_status", "fail"), + "promotion_status": promotion.get("promotion_status", "blocked"), + "source_count": len({card.get("source_id") for card in cards if card.get("source_id")}), + "card_count": len(cards), + "weak_count": confidence_counts.get("weak", 0), + "unsupported_count": confidence_counts.get("unsupported", 0), + "retrieval_qa_passed": retrieval_report.get("passed", 0), + "retrieval_qa_failed": retrieval_report.get("failed", 0), + "review_required_count": review_status_counts.get("review_required", 0), + "duplicate_conflicts": dedupe_report.get("duplicate_conflicts", 0), + "production_ready": bool(promotion.get("production_ready", False)), + "status": "pass" if promotion.get("production_ready", False) else "blocked", + "blocker_reasons": promotion.get("blocking_reasons", []), + "next_action": "governed release ready" if promotion.get("production_ready", False) else "resolve promotion blockers", + } + write_json(RELEASE_MANIFEST_JSON, manifest) + weak_backlog = build_weak_evidence_backlog(cards, review_queue) + write_json(WEAK_EVIDENCE_BACKLOG, weak_backlog) + audit_snapshot = build_release_audit_snapshot(manifest, retrieval_report, card_report, review_queue) + write_json(RELEASE_AUDIT_SNAPSHOT, audit_snapshot) + governance_state = { + "generated_at": utc_now(), + "release_manifest": manifest, + "acceptance": acceptance, + "promotion": promotion, + "weak_evidence_backlog": { + "path": "publish/weak_evidence_backlog.json", + "weak_count": weak_backlog["weak_count"], + }, + "release_audit_snapshot": { + "path": "publish/release_audit_snapshot.json", + "production_ready": audit_snapshot["production_ready"], + }, + "optional_routes": { + "ollama": retrieval_report.get("ollama", {}), + "gemini": retrieval_report.get("gemini", {}), + }, + } + write_json(GOVERNANCE_STATE, governance_state) + (COMPACT_DIR / "KB__RELEASE_MANIFEST.md").write_text(render_release_markdown(manifest), encoding="utf-8") + (COMPACT_DIR / "KB__REVIEW_QUEUE.md").write_text(render_review_queue_markdown(review_queue), encoding="utf-8") + print(f"acceptance_status={manifest['acceptance_status']}") + print(f"promotion_status={manifest['promotion_status']}") + print(f"production_ready={str(manifest['production_ready']).lower()}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/build_synthesis_layer.py b/scripts/build_synthesis_layer.py new file mode 100644 index 0000000..d7c8c0b --- /dev/null +++ b/scripts/build_synthesis_layer.py @@ -0,0 +1,909 @@ +#!/usr/bin/env python3 +import argparse +import re +import sys +from collections import Counter +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_OUTPUT = Path("publish/chatgpt_project_compact") + +COMPACT_PACKAGE_FILES = [ + "KB__00_INDEX.md", + "KB__01_NAVIGATION.md", + "KB__02_CONTENT.md", + "KB__03_WORKFLOWS_TRACEABILITY.md", + "KB__04_SMOKE_QA.md", + "KB__05_CANONICAL_CONCEPTS.md", + "KB__06_OPERATIONAL_FRAMEWORKS.md", + "KB__07_PATTERNS_AND_FAILURES.md", + "KB__08_USE_CASES_FOR_SERGEY.md", + "KB__RELEASE_MANIFEST.md", + "KB__CHANGELOG.md", + "KB__REVIEW_QUEUE.md", + "KB__CARD_SCHEMA.md", + "KB__CONFIDENCE_RULES.md", + "KB__PROMOTION_GATES.md", + "KB__RETRIEVAL_QA.md", + "KB__DEDUPLICATION.md", + "KB__USE_CASE_ROUTING.md", + "README.md", + "MANIFEST.md", + "SYNTHESIS_MANIFEST.md", +] + +INPUT_DIRS = [ + ROOT / "workspace" / "source_cards", + ROOT / "workspace" / "clean_notes", + ROOT / "publish" / "markdown_kb", + ROOT / "publish" / "chatgpt_project", +] + +FORBIDDEN_INPUT_PARTS = ( + "workspace/raw", + "workspace/chunks", + "logs", + "temp", + "embeddings", + "vector_db", +) +FORBIDDEN_OUTPUT_MARKERS = ( + "workspace/raw", + "input/raw", + "workspace/chunks", + ".txt", +) + +OUTPUT_FILES = [ + "KB__05_CANONICAL_CONCEPTS.md", + "KB__06_OPERATIONAL_FRAMEWORKS.md", + "KB__07_PATTERNS_AND_FAILURES.md", + "KB__08_USE_CASES_FOR_SERGEY.md", + "KB__RELEASE_MANIFEST.md", + "KB__REVIEW_QUEUE.md", + "KB__USE_CASE_ROUTING.md", + "SYNTHESIS_MANIFEST.md", +] + +REQUIRED_CONCEPTS = [ + "AI Systems", + "Knowledge Organization", + "Source Card", + "ChatGPT Project Package", + "Publish Package", + "Consumer QA", + "Traceability", + "Workflow Extraction", + "Knowledge Distillation", + "Canonical Concepts", + "Operational Framework", + "AI OS", + "Codex Workflow", + "Judge / Reviewer", + "Prompt Routing", + "Context Engineering", + "File-first Knowledge", + "Open WebUI Retrieval", + "RAG", + "Finance Analytics", + "Power BI Analytics", + "Security Workflow", +] + +REQUIRED_FRAMEWORKS = [ + "Source-to-KB Pipeline", + "ChatGPT Project Upload Workflow", + "Consumer QA Workflow", + "Traceability Check Workflow", + "Concept Synthesis Workflow", + "Open WebUI Retrieval Workflow", + "ChatGPT + Open WebUI + Codex Workflow", + "AI OS Knowledge Loop", + "Finance Analytics KB Workflow", + "Codex Task Preparation Workflow", +] + +REQUIRED_PATTERNS = [ + "File-first knowledge", + "Routing before reasoning", + "Traceability before automation", + "Compact package over raw dump", + "Open WebUI for retrieval, ChatGPT for reasoning", + "Codex for implementation", + "Smoke tests before scaling", + "Synthesis before embeddings", +] + +REQUIRED_ANTI_PATTERNS = [ + "Uploading raw transcripts directly", + "Using source cards as primary user-facing KB", + "Adding embeddings before structure works", + "Treating QA recommendations as source facts", + "Answering without evidence", + "One giant noisy markdown file", + "Mixing navigation and raw content", + "Inventing definitions where source says not found", +] + +REQUIRED_USE_CASES = [ + "Prepare Codex task from retrieved KB chunks", + "Build financial analytics documentation from KB", + "Create AI OS workflow map", + "Audit a KB package before upload", + "Turn Open WebUI retrieval into ChatGPT synthesis", + "Build project-specific compact context package", + "Prepare QA questions for a new knowledge project", + "Build reusable prompt library from source cards", + "Create operational documentation for finance/audit automation", + "Decide when to add embeddings / RAG / Qdrant", +] + + +@dataclass +class SourceDoc: + path: Path + rel_path: str + text: str + source_id: str + priority: int + title: str = "" + core_topic: str = "" + concepts: list[str] = field(default_factory=list) + workflows: list[str] = field(default_factory=list) + rules: list[str] = field(default_factory=list) + risks: list[str] = field(default_factory=list) + + +@dataclass +class Evidence: + label: str + fact: str + + +@dataclass +class BuildState: + timestamp: str + docs: list[SourceDoc] + warnings: list[str] = field(default_factory=list) + validation: list[tuple[str, str]] = field(default_factory=list) + evidence_quality: Counter = field(default_factory=Counter) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build deterministic KB synthesis layer.") + parser.add_argument( + "--max-sources-per-concept", + type=int, + default=20, + help="Maximum evidence sources listed for each concept/framework block.", + ) + parser.add_argument( + "--output", + default=str(DEFAULT_OUTPUT), + help="Output directory for synthesis layer files.", + ) + parser.add_argument( + "--use-llm", + action="store_true", + help="Reserved optional mode. Default deterministic mode is always used in this script.", + ) + return parser.parse_args() + + +def progress(message: str) -> None: + print(f"Progress: {message}") + + +def fail(message: str) -> None: + print(f"error: {message}", file=sys.stderr) + raise SystemExit(1) + + +def rel(path: Path) -> str: + try: + return path.resolve().relative_to(ROOT).as_posix() + except ValueError: + return path.as_posix() + + +def read_text(path: Path) -> str | None: + try: + return path.read_text(encoding="utf-8") + except UnicodeDecodeError: + return None + + +def clean_line(line: str) -> str: + line = re.sub(r"`chunk_\d+`", "", line) + line = re.sub(r"\s+", " ", line).strip() + line = re.sub(r"^-+\s*", "", line) + line = re.sub(r"^\d+\.\s*", "", line) + line = line.replace("**", "") + return line.strip() + + +def sanitize_output(text: str) -> str: + lines: list[str] = [] + for line in text.splitlines(): + if any(marker in line for marker in FORBIDDEN_OUTPUT_MARKERS): + continue + lines.append(line.rstrip()) + compact: list[str] = [] + blank = False + for line in lines: + if line: + compact.append(line) + blank = False + elif not blank: + compact.append("") + blank = True + return "\n".join(compact).rstrip() + "\n" + + +def section(text: str, heading: str) -> list[str]: + lines = text.splitlines() + start = None + for idx, line in enumerate(lines): + if line.strip().lower() == f"## {heading}".lower(): + start = idx + 1 + break + if start is None: + return [] + end = len(lines) + for idx in range(start, len(lines)): + if lines[idx].startswith("## "): + end = idx + break + values = [] + for line in lines[start:end]: + item = clean_line(line) + if item and not item.startswith("#"): + values.append(item) + return values[:12] + + +def metadata_value(text: str, key: str) -> str: + pattern = re.compile(rf"^- {re.escape(key)}:\s*(.+)$", re.MULTILINE) + match = pattern.search(text) + return clean_line(match.group(1)) if match else "" + + +def source_id_for(path: Path, text: str) -> str: + return metadata_value(text, "source_id") or path.stem.split(".")[0] + + +def discover_sources() -> list[SourceDoc]: + docs: list[SourceDoc] = [] + for priority, folder in enumerate(INPUT_DIRS): + if not folder.is_dir(): + continue + for path in sorted(folder.iterdir(), key=lambda item: item.name.lower()): + if not path.is_file() or path.name.startswith("."): + continue + if any(part in rel(path) for part in FORBIDDEN_INPUT_PARTS): + continue + if path.suffix.lower() not in {".md", ".csv"}: + continue + text = read_text(path) + if text is None: + continue + docs.append( + SourceDoc( + path=path, + rel_path=rel(path), + text=text, + source_id=source_id_for(path, text), + priority=priority, + title=metadata_value(text, "title") or path.stem, + core_topic=" ".join(section(text, "Core Topic")[:2]), + concepts=section(text, "Key Concepts") + section(text, "Concepts"), + workflows=section(text, "Procedures / Workflows") + section(text, "Procedures / Workflow"), + rules=section(text, "Practical Rules") + section(text, "Main Ideas"), + risks=section(text, "Risks / Caveats") + section(text, "Not Found / Unclear"), + ) + ) + return docs + + +def tokenize(value: str) -> set[str]: + words = re.findall(r"[A-Za-zА-Яа-я0-9]+", value.lower()) + return {word for word in words if len(word) >= 3} + + +def evidence_for(name: str, docs: list[SourceDoc], limit: int) -> list[Evidence]: + tokens = tokenize(name) + scored: list[tuple[int, int, SourceDoc]] = [] + for doc in docs: + haystack = " ".join( + [doc.rel_path, doc.title, doc.core_topic] + + doc.concepts + + doc.workflows + + doc.rules + + doc.risks + ).lower() + score = sum(1 for token in tokens if token in haystack) + if score: + scored.append((-score, doc.priority, doc)) + result: list[Evidence] = [] + seen: set[str] = set() + for _, _, doc in sorted(scored, key=lambda item: (item[0], item[1], item[2].rel_path)): + if doc.rel_path in seen: + continue + seen.add(doc.rel_path) + fact = first_fact(doc, tokens) + result.append(Evidence(label=f"{doc.source_id} / {Path(doc.rel_path).name}", fact=fact)) + if len(result) >= limit: + break + return result + + +def first_fact(doc: SourceDoc, tokens: set[str]) -> str: + candidates = [doc.core_topic] + doc.concepts + doc.workflows + doc.rules + doc.risks + for item in candidates: + lower = item.lower() + if any(token in lower for token in tokens): + return clean_line(item)[:280] + for item in candidates: + if item: + return clean_line(item)[:280] + return f"Source file references this area: {Path(doc.rel_path).name}" + + +def confidence(evidence: list[Evidence]) -> str: + if len(evidence) >= 5: + return "strong" + if len(evidence) >= 2: + return "medium" + if len(evidence) == 1: + return "weak" + return "unsupported" + + +def evidence_lines(evidence: list[Evidence], confidence_value: str) -> str: + if not evidence: + return "- source: Evidence is weak / not enough source support.\n- confidence: unsupported" + rows = [f"- source: {item.label} — {item.fact}" for item in evidence] + rows.append(f"- confidence: {confidence_value}") + return "\n".join(rows) + + +def candidate_concepts(docs: list[SourceDoc]) -> list[str]: + counter: Counter[str] = Counter() + for doc in docs: + for line in doc.concepts: + match = re.match(r"([^:：-]+)", clean_line(line)) + if match: + candidate = match.group(1).strip(" -*`") + if 3 <= len(candidate) <= 60: + counter[candidate] += 1 + for line in doc.text.splitlines(): + if line.startswith("## ") or line.startswith("### "): + heading = line.strip("# ").strip() + if 3 <= len(heading) <= 60: + counter[heading] += 1 + concepts = list(REQUIRED_CONCEPTS) + for name, _ in counter.most_common(40): + if name not in concepts and len(concepts) < 50: + concepts.append(name) + return concepts[:50] + + +def concept_definition(name: str, evidence: list[Evidence]) -> str: + if not evidence: + return "TODO: Evidence is weak / not enough source support." + return f"In this KB, `{name}` is represented by source-backed notes around: {evidence[0].fact}" + + +def build_concepts(state: BuildState, max_sources: int) -> str: + parts = ["# Canonical Concepts", ""] + for name in candidate_concepts(state.docs): + evidence = evidence_for(name, state.docs, max_sources) + conf = confidence(evidence) + state.evidence_quality[conf] += 1 + first = evidence[0].fact if evidence else "Evidence is weak / not enough source support." + parts.extend( + [ + f"## Concept: {name}", + "", + "### Definition", + concept_definition(name, evidence), + "", + "### Why it matters", + f"This concept is useful when organizing KB retrieval, operational workflows, or AI/analytics work. Confidence: {conf}.", + "", + "### Source-backed facts", + f"- {first}", + "", + "### Interpretation", + "- Use this as a working synthesis only where the evidence list supports it.", + "", + "### Operational use", + f"- Use `{name}` as a retrieval, planning, QA, or workflow label when matching project material to source-backed evidence.", + "", + "### Related concepts", + "- Traceability", + "- Knowledge Organization", + "- Operational Framework", + "", + "### Anti-patterns", + "- Inventing definitions where source says not found.", + "- Treating weak evidence as a finished concept.", + "", + "### Limitations", + "- Evidence may be uneven across sources.", + "- Weak or unsupported concepts remain blocked until automated review queue and retrieval QA clear them.", + "", + "### Evidence", + evidence_lines(evidence, conf), + "", + ] + ) + return "\n".join(parts) + + +FRAMEWORK_STEPS = { + "Source-to-KB Pipeline": ["Inventory source files.", "Create chunks.", "Create clean notes.", "Create source cards.", "Build KB outputs.", "Run QA and publish."], + "ChatGPT Project Upload Workflow": ["Build compact package.", "Upload compact files.", "Start with the index.", "Run smoke questions.", "Record retrieval gaps."], + "Consumer QA Workflow": ["Select smoke questions.", "Ask against uploaded KB.", "Check source grounding.", "Record pass/fail.", "Revise package when needed."], + "Traceability Check Workflow": ["Find answer claim.", "Locate source evidence.", "Check source identifier.", "Flag unsupported claims.", "Record confidence."], + "Concept Synthesis Workflow": ["Collect candidate concepts.", "Match evidence.", "Draft deterministic skeleton.", "Mark weak evidence.", "Write unresolved items to review queue."], + "Open WebUI Retrieval Workflow": ["Retrieve chunks in Open WebUI.", "Copy retrieved excerpts.", "Ask ChatGPT to synthesize from excerpts.", "Check evidence.", "Update notes."], + "ChatGPT + Open WebUI + Codex Workflow": ["Retrieve evidence.", "Synthesize task brief.", "Write SPEC/plan/tasks.", "Run Codex implementation.", "Validate outputs."], + "AI OS Knowledge Loop": ["Capture source.", "Distill knowledge.", "Package for retrieval.", "Use in workflows.", "Review and improve."], + "Finance Analytics KB Workflow": ["Collect source docs.", "Separate facts from assumptions.", "Document metrics.", "Run deterministic checks.", "Publish guidance."], + "Codex Task Preparation Workflow": ["Retrieve KB context.", "Draft task spec.", "Build plan.", "Lock scope.", "Execute and validate."], +} + + +def build_frameworks(state: BuildState, max_sources: int) -> str: + parts = ["# Operational Frameworks", ""] + for name in REQUIRED_FRAMEWORKS: + evidence = evidence_for(name, state.docs, max_sources) + conf = confidence(evidence) + state.evidence_quality[conf] += 1 + steps = FRAMEWORK_STEPS[name] + parts.extend( + [ + f"## Framework: {name}", + "", + "### Purpose", + f"Provide a reusable operating sequence for `{name}` without treating weak evidence as finished synthesis.", + "", + "### Trigger", + f"Use when work matches `{name}` and source-backed evidence is needed before action.", + "", + "### Inputs", + "- Source cards, clean notes, published markdown, or compact package files.", + "- Retrieved excerpts or file references.", + "", + "### Ordered steps", + *[f"{idx}. {step}" for idx, step in enumerate(steps, 1)], + "", + "### Outputs", + "- A grounded workflow result, package, QA note, task brief, or documentation block.", + "", + "### QA gates", + "- Evidence is listed.", + "- Confidence is labeled.", + "- Weak evidence is marked as weak.", + "- Unsupported claims are not promoted to facts.", + "", + "### Failure modes", + "- Evidence is weak / not enough source support.", + "- Source references are missing.", + "- Navigation, raw content, and synthesis are mixed.", + "", + "### How Sergey should use it", + "- Use this framework as a checklist before asking ChatGPT or Codex to produce final work.", + "", + "### Facts", + f"- Evidence matches found: {len(evidence)}.", + "", + "### Interpretation", + "- This framework is deterministic scaffolding and remains blocked where evidence is weak.", + "", + "### Operational use", + "- Apply the ordered steps and QA gates directly to the matching workflow.", + "", + "### Limitations", + "- Convert from skeleton to production-approved operating guide only after automated acceptance gates pass.", + "", + "### Evidence", + evidence_lines(evidence, conf), + "", + ] + ) + return "\n".join(parts) + + +def pattern_block(kind: str, name: str, state: BuildState, max_sources: int) -> list[str]: + evidence = evidence_for(name, state.docs, max_sources) + conf = confidence(evidence) + state.evidence_quality[conf] += 1 + return [ + f"## {kind}: {name}", + "", + "### Meaning", + f"`{name}` is a reusable rule extracted as deterministic guidance. Evidence quality: {conf}.", + "", + "### When to use", + "- Use when the current task matches the named retrieval, QA, synthesis, or implementation situation.", + "", + "### Why it works", + "- It keeps source evidence, workflow routing, and generated output separated.", + "", + "### Risk if ignored", + "- Evidence may be lost, noisy raw material may leak into user-facing output, or unsupported claims may be treated as facts.", + "", + "### Facts", + f"- Evidence matches found: {len(evidence)}.", + "", + "### Interpretation", + "- Treat as a working rule unless evidence is strong.", + "", + "### Operational use", + "- Apply as a check before packaging, retrieval, synthesis, or implementation.", + "", + "### Limitations", + "- Evidence is weak / not enough source support." if conf in {"weak", "unsupported"} else "- Source support exists but still needs automated acceptance before production use.", + "", + "### Evidence", + evidence_lines(evidence, conf), + "", + ] + + +def build_patterns(state: BuildState, max_sources: int) -> str: + parts = ["# Patterns", ""] + for name in REQUIRED_PATTERNS: + parts.extend(pattern_block("Pattern", name, state, max_sources)) + parts.extend(["# Anti-patterns / Failure Modes", ""]) + for name in REQUIRED_ANTI_PATTERNS: + parts.extend(pattern_block("Anti-pattern", name, state, max_sources)) + return "\n".join(parts) + + +def build_use_cases(state: BuildState, max_sources: int) -> str: + parts = ["# Use Cases For Sergey", ""] + for name in REQUIRED_USE_CASES: + evidence = evidence_for(name, state.docs, max_sources) + conf = confidence(evidence) + state.evidence_quality[conf] += 1 + parts.extend( + [ + f"## Use Case: {name}", + "", + "### Goal", + f"Use source-backed KB material to perform `{name}` with explicit evidence and confidence.", + "", + "### Inputs", + "- Retrieved KB excerpts.", + "- Source card or clean note references.", + "- Compact package navigation files when relevant.", + "", + "### Steps", + "1. Collect the relevant retrieved excerpts.", + "2. List evidence filenames before synthesis.", + "3. Separate facts from interpretation.", + "4. Produce the requested output.", + "5. Mark weak or missing evidence explicitly.", + "", + "### Output", + "- A grounded task brief, workflow note, QA checklist, synthesis, or documentation block.", + "", + "### Prompt template", + "```text", + f"Use only the provided KB excerpts to work on: {name}.", + "Separate Facts, Interpretation, Operational use, Limitations, Evidence, and Confidence.", + "If evidence is weak, write: Evidence is weak / not enough source support.", + "```", + "", + "### Risks", + "- Weak evidence may lead to overconfident synthesis.", + "- Retrieved chunks may omit necessary context.", + "", + "### Facts", + f"- Evidence matches found: {len(evidence)}.", + "", + "### Interpretation", + "- This use case is a deterministic operating skeleton, not a claim of complete synthesis.", + "", + "### Operational use", + "- Use the prompt template and evidence checklist before producing final work.", + "", + "### Limitations", + "- Refine with deterministic usage examples after automated acceptance evidence exists.", + "", + "### Evidence", + evidence_lines(evidence, conf), + "", + ] + ) + return "\n".join(parts) + + +def promotion_blocked(state: BuildState) -> bool: + return state.evidence_quality.get("unsupported", 0) > 0 + + +def build_release_manifest(state: BuildState, output_dir: Path) -> str: + inputs = "\n".join(f"- `{rel(path)}`" for path in INPUT_DIRS if path.exists()) or "- None" + outputs = "\n".join(f"- `{rel(output_dir / name)}`" for name in COMPACT_PACKAGE_FILES) + unsupported = state.evidence_quality.get("unsupported", 0) + weak = state.evidence_quality.get("weak", 0) + acceptance_status = "blocked" if unsupported else "pass_with_residual_risks" + promoted = "no" + return f"""# KB Release Manifest + +## kb_version +managed-knowledge-system-v1 + +## build_date +{state.timestamp} + +## source_inputs +{inputs} + +## processed_transcripts +- Determined by upstream inventory/source-card pipeline. +- This synthesis layer uses allowed source cards, clean notes, markdown KB, and ChatGPT package files. + +## generated_cards +- Source Card +- Concept Card +- Workflow Card +- Pattern Card +- QA Card +- Navigation Card +- Use Case Card + +## publish_outputs +- `publish/chatgpt_project/` +- `publish/markdown_kb/` + +## compact_package_outputs +{outputs} + +## smoke_qa_status +automated_schema_present + +## acceptance_status +{acceptance_status} + +## residual_risks +- weak_items: {weak} +- unsupported_items: {unsupported} +- weak and unsupported items remain blocked from production promotion. + +## blocked_items +- embeddings +- semantic search +- vector DB +- web UI +- agentic workflows +- autonomous retrieval + +## next_scope +- Clear automated review queue items before promotion. +- Keep retrieval QA stable before adding retrieval infrastructure. + +## promoted_to_production +{promoted} +""" + + +def build_review_queue(state: BuildState) -> str: + weak = state.evidence_quality.get("weak", 0) + unsupported = state.evidence_quality.get("unsupported", 0) + status = "blocked" if unsupported else "review_required" if weak else "clear" + return f"""# KB Review Queue + +## Queue status +- review_status: {status} +- weak_items: {weak} +- unsupported_items: {unsupported} + +## Automated review classes +- weak cards +- unsupported claims +- duplicate candidates +- conflicting concepts +- unresolved workflows +- missing traceability +- stale concepts +- low-confidence synthesis + +## Current unresolved items +- weak synthesis items: {weak} +- unsupported synthesis items: {unsupported} +- duplicate candidates: tracked by `KB__DEDUPLICATION.md` +- conflicting concepts: tracked by `KB__DEDUPLICATION.md` + +## Promotion impact +- Unsupported items block promotion. +- Weak items are allowed only when listed here and excluded from production-approved claims. +- Deprecated or duplicate candidates are not silently deleted. +""" + + +def build_use_case_routing(state: BuildState) -> str: + weak = state.evidence_quality.get("weak", 0) + unsupported = state.evidence_quality.get("unsupported", 0) + routes = "\n".join( + f"- {route}" + for route in ( + "analytical memo", + "FP&A", + "counterparty audit", + "Power BI", + "Codex tasks", + "AI OS architecture", + "model routing", + "QA / review", + "governance", + "automation", + ) + ) + return f"""# Use Case Layer + +## Routing contract +Every strong concept, workflow, or pattern must map to an operational use case before production use. + +## Required routes +{routes} + +## Use Case Fields +- use_case_id +- applicable_roles +- required_confidence +- required_sources +- workflows_used +- operational_risks +- recommended_models + +## Automated routing rules +- Operational recommendations require `medium` or `strong` confidence. +- Weak use cases remain in `KB__REVIEW_QUEUE.md`. +- Unsupported use cases block promotion. +- Recommended models must be described as routing guidance, not as source facts. + +## User aliases +- `7 великих на GitHub` = Hamel Husain (`ba8f38829c4c`), Simon Willison (`047131d56e3f`), Sebastian Raschka (`0289a622670f`), Paul Gauthier (`1b55b74e10a7`), Jason Liu (`6d3f1535b610`), Georgi Gerganov (`fbca6860f502`), Phil Wang (`3aab3b710e61`). +- Route this alias to the seven source cards above before using the wider KB. + +## Current routing status +- weak_items: {weak} +- unsupported_items: {unsupported} +- promoted_to_production: no +""" + + +def build_manifest(state: BuildState, output_dir: Path) -> str: + inputs = "\n".join(f"- `{rel(path)}`" for path in INPUT_DIRS if path.exists()) or "- None" + outputs = "\n".join(f"- `{rel(output_dir / name)}`" for name in OUTPUT_FILES) + warnings = "\n".join(f"- {item}" for item in state.warnings) or "- None" + validation = "\n".join(f"- {name}: {status}" for name, status in state.validation) or "- Not run" + quality = "\n".join( + f"- {name}: {state.evidence_quality.get(name, 0)}" + for name in ("strong", "medium", "weak", "unsupported") + ) + return f"""# Synthesis Manifest + +## Generation timestamp +{state.timestamp} + +## Inputs used +{inputs} + +## Outputs created +{outputs} + +## Concepts created +- {len(candidate_concepts(state.docs))} + +## Frameworks created +- {len(REQUIRED_FRAMEWORKS)} + +## Patterns created +- {len(REQUIRED_PATTERNS)} + +## Anti-patterns created +- {len(REQUIRED_ANTI_PATTERNS)} + +## Evidence quality summary +{quality} + +## Warnings +{warnings} + +## Validation status +{validation} +""" + + +def write_output(path: Path, text: str) -> None: + path.write_text(sanitize_output(text), encoding="utf-8") + + +def validate_outputs(output_dir: Path, state: BuildState) -> None: + checks: list[tuple[str, bool]] = [] + files = [output_dir / name for name in OUTPUT_FILES] + checks.append(("all synthesis output files exist", all(path.is_file() for path in files))) + utf8_ok = True + content = "" + for path in files: + try: + text = path.read_text(encoding="utf-8") + except UnicodeDecodeError: + utf8_ok = False + text = "" + content += "\n" + text + checks.append(("outputs are UTF-8 readable", utf8_ok)) + checks.append(("concept evidence and confidence present", "## Concept:" in content and "- confidence:" in (output_dir / "KB__05_CANONICAL_CONCEPTS.md").read_text(encoding="utf-8"))) + frameworks = (output_dir / "KB__06_OPERATIONAL_FRAMEWORKS.md").read_text(encoding="utf-8") + checks.append(("framework required sections present", all(marker in frameworks for marker in ("### Trigger", "### Inputs", "### Ordered steps", "### Outputs", "### QA gates")))) + checks.append(("required anti-pattern present", "Inventing definitions where source says not found" in content)) + checks.append(("raw transcript markers absent", not any(marker in content for marker in FORBIDDEN_OUTPUT_MARKERS))) + release = (output_dir / "KB__RELEASE_MANIFEST.md").read_text(encoding="utf-8") + checks.append(("release manifest governance fields present", all(marker in release for marker in ("kb_version", "acceptance_status", "promoted_to_production")))) + review_queue = (output_dir / "KB__REVIEW_QUEUE.md").read_text(encoding="utf-8") + checks.append(("review queue blocks unsupported items", "Unsupported items block promotion" in review_queue)) + use_case_routing = (output_dir / "KB__USE_CASE_ROUTING.md").read_text(encoding="utf-8") + checks.append(("use-case routing fields present", all(marker in use_case_routing for marker in ("use_case_id", "required_confidence", "recommended_models")))) + manifest = (output_dir / "SYNTHESIS_MANIFEST.md").read_text(encoding="utf-8") + checks.append(("manifest warnings and evidence summary present", "## Warnings" in manifest and "## Evidence quality summary" in manifest)) + for name, ok in checks: + state.validation.append((name, "pass" if ok else "fail")) + failed = [name for name, ok in checks if not ok] + if failed: + fail("validation failed: " + ", ".join(failed)) + + +def main() -> int: + args = parse_args() + output_dir = Path(args.output) + if not output_dir.is_absolute(): + output_dir = ROOT / output_dir + timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds") + + if args.use_llm: + print("warning: --use-llm is reserved; deterministic fallback mode will be used.", file=sys.stderr) + + progress("discovering source evidence") + docs = discover_sources() + if not docs: + fail("no allowed sources found") + + state = BuildState(timestamp=timestamp, docs=docs) + if args.use_llm: + state.warnings.append("--use-llm requested, but deterministic fallback was used.") + state.warnings.append( + "Default output is deterministic synthesis scaffolding; weak evidence is marked instead of polished as fact." + ) + + progress("preparing output folder") + output_dir.mkdir(parents=True, exist_ok=True) + + progress("writing synthesis files") + write_output(output_dir / "KB__05_CANONICAL_CONCEPTS.md", build_concepts(state, args.max_sources_per_concept)) + write_output(output_dir / "KB__06_OPERATIONAL_FRAMEWORKS.md", build_frameworks(state, args.max_sources_per_concept)) + write_output(output_dir / "KB__07_PATTERNS_AND_FAILURES.md", build_patterns(state, args.max_sources_per_concept)) + write_output(output_dir / "KB__08_USE_CASES_FOR_SERGEY.md", build_use_cases(state, args.max_sources_per_concept)) + write_output(output_dir / "KB__RELEASE_MANIFEST.md", build_release_manifest(state, output_dir)) + write_output(output_dir / "KB__REVIEW_QUEUE.md", build_review_queue(state)) + write_output(output_dir / "KB__USE_CASE_ROUTING.md", build_use_case_routing(state)) + write_output(output_dir / "SYNTHESIS_MANIFEST.md", build_manifest(state, output_dir)) + + progress("validating synthesis layer") + validate_outputs(output_dir, state) + write_output(output_dir / "SYNTHESIS_MANIFEST.md", build_manifest(state, output_dir)) + + progress(f"done: {rel(output_dir)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_acceptance_gate.py b/scripts/run_acceptance_gate.py new file mode 100644 index 0000000..51166e3 --- /dev/null +++ b/scripts/run_acceptance_gate.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + +from notes_to_kb.governance import ( # noqa: E402 + ACCEPTANCE_REPORT, + CARD_VALIDATION_REPORT, + COMPACT_DIR, + DEDUPLICATION_REPORT, + REVIEW_QUEUE_JSON, + RETRIEVAL_QA_RESULTS, + read_json, + utc_now, + write_json, +) + + +REQUIRED_COMPACT_FILES = [ + "KB__00_INDEX.md", + "KB__01_NAVIGATION.md", + "KB__02_CONTENT.md", + "KB__03_WORKFLOWS_TRACEABILITY.md", + "KB__04_SMOKE_QA.md", + "KB__RELEASE_MANIFEST.md", + "KB__REVIEW_QUEUE.md", + "KB__PROMOTION_GATES.md", + "KB__RETRIEVAL_QA.md", +] + + +def main() -> int: + card_report = read_json(CARD_VALIDATION_REPORT, {}) + dedupe_report = read_json(DEDUPLICATION_REPORT, {}) + retrieval_report = read_json(RETRIEVAL_QA_RESULTS, {}) + review_queue = read_json(REVIEW_QUEUE_JSON, {"items": []}) + missing_files = [name for name in REQUIRED_COMPACT_FILES if not (COMPACT_DIR / name).exists()] + confidence_counts = card_report.get("confidence_counts", {}) + unsupported_count = confidence_counts.get("unsupported", 0) + weak_count = confidence_counts.get("weak", 0) + high_severity = sum(1 for item in review_queue.get("items", []) if item.get("severity") == "high") + retrieval_failed = retrieval_report.get("failed", 0) + duplicate_conflicts = dedupe_report.get("duplicate_conflicts", 0) + schema_errors = card_report.get("schema_invalid_count", 0) + + blocking_reasons = [] + if missing_files: + blocking_reasons.append("required_files_missing") + if schema_errors: + blocking_reasons.append("card_schema_critical_errors") + if unsupported_count: + blocking_reasons.append("unsupported_production_items") + if retrieval_failed: + blocking_reasons.append("retrieval_qa_failed") + if high_severity: + blocking_reasons.append("high_severity_review_items") + if duplicate_conflicts: + blocking_reasons.append("duplicate_conflicts_unresolved") + + status = "pass" if not blocking_reasons else "fail" + report = { + "generated_at": utc_now(), + "command": "python3 scripts/run_acceptance_gate.py", + "mode": "runtime_governance", + "input_paths": [ + "publish/card_validation_report.json", + "publish/deduplication_report.json", + "publish/retrieval_qa_results.json", + "publish/review_queue.json", + "publish/chatgpt_project_compact", + ], + "acceptance_status": status, + "status": status, + "blocking_reasons": blocking_reasons, + "missing_files": missing_files, + "weak_count": weak_count, + "unsupported_count": unsupported_count, + "high_severity_review_items": high_severity, + "retrieval_qa_failed": retrieval_failed, + "duplicate_conflicts": duplicate_conflicts, + "schema_invalid_count": schema_errors, + "next_action": "run promotion gate" if status == "pass" else "resolve blocking reason codes", + } + write_json(ACCEPTANCE_REPORT, report) + print(f"acceptance_status={status}") + print(f"blocking_reasons={','.join(blocking_reasons) if blocking_reasons else 'none'}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_chunking.py b/scripts/run_chunking.py new file mode 100644 index 0000000..8ed34c7 --- /dev/null +++ b/scripts/run_chunking.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from notes_to_kb.chunking import build_chunks +from notes_to_kb.paths import CHUNKS_DIR, DEFAULT_INPUT_RAW + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--input", default=str(DEFAULT_INPUT_RAW)) + parser.add_argument("--output-dir", default=str(CHUNKS_DIR)) + parser.add_argument("--chunk-size", type=int, default=6000) + args = parser.parse_args() + + build_chunks(Path(args.input), Path(args.output_dir), args.chunk_size) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_clean_note.py b/scripts/run_clean_note.py new file mode 100644 index 0000000..06c5c65 --- /dev/null +++ b/scripts/run_clean_note.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from notes_to_kb.clean_note import generate_all_clean_notes, generate_clean_note +from notes_to_kb.llm_client import build_llm_client +from notes_to_kb.paths import CLEAN_NOTE_PROMPT, CLEAN_NOTES_DIR, CHUNKS_DIR, REVIEW_QUEUE, SOURCES_CSV + + +def main() -> None: + parser = argparse.ArgumentParser() + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--source-id") + group.add_argument("--all", action="store_true") + parser.add_argument("--provider", default="mock") + parser.add_argument("--model", default="") + parser.add_argument("--ollama-base-url", default=None) + parser.add_argument("--base-url", default=None) + parser.add_argument("--timeout", type=float, default=120.0) + parser.add_argument("--sources-index", default=str(SOURCES_CSV)) + parser.add_argument("--chunks-dir", default=str(CHUNKS_DIR)) + parser.add_argument("--output-dir", default=str(CLEAN_NOTES_DIR)) + parser.add_argument("--review-queue", default=str(REVIEW_QUEUE)) + parser.add_argument("--prompt", default=str(CLEAN_NOTE_PROMPT)) + parser.add_argument("--overwrite", action="store_true") + args = parser.parse_args() + + client = build_llm_client( + args.provider, + model=args.model, + base_url=args.base_url or args.ollama_base_url, + timeout=args.timeout, + ) + if args.all: + generate_all_clean_notes( + sources_index=Path(args.sources_index), + chunks_root=Path(args.chunks_dir), + output_dir=Path(args.output_dir), + review_queue_path=Path(args.review_queue), + client=client, + prompt_path=Path(args.prompt), + overwrite=args.overwrite, + progress=True, + ) + else: + generate_clean_note( + source_id=args.source_id, + sources_index=Path(args.sources_index), + chunks_root=Path(args.chunks_dir), + output_dir=Path(args.output_dir), + review_queue_path=Path(args.review_queue), + client=client, + prompt_path=Path(args.prompt), + overwrite=args.overwrite, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_deduplication.py b/scripts/run_deduplication.py new file mode 100644 index 0000000..c6086e5 --- /dev/null +++ b/scripts/run_deduplication.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +import argparse +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + +from notes_to_kb.governance import ( # noqa: E402 + CARD_VALIDATION_REPORT, + DEDUPLICATION_REPORT, + deduplicate_cards, + merge_review_items, + optional_route_status, + read_json, + utc_now, + write_json, +) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run deterministic card deduplication.") + parser.add_argument("--use-ollama", choices=["auto", "on", "off"], default="auto") + args = parser.parse_args() + + card_report = read_json(CARD_VALIDATION_REPORT, {"cards": []}) + cards = card_report.get("cards", []) + duplicates, review_items = deduplicate_cards(cards) + review_queue = merge_review_items(review_items) + route_status = optional_route_status(use_ollama=args.use_ollama, use_gemini="off") + report = { + "generated_at": utc_now(), + "command": f"python3 scripts/run_deduplication.py --use-ollama {args.use_ollama}", + "mode": "runtime_governance", + "input_paths": ["publish/card_validation_report.json"], + "card_count": len(cards), + "duplicate_count": len(duplicates), + "duplicate_conflicts": len([item for item in duplicates if item["merge_action"] == "review_required"]), + "duplicates": duplicates, + "ollama": route_status["ollama"], + "status": "pass" if not duplicates else "fail", + "blocker_reasons": ["duplicate_conflicts_unresolved"] if duplicates else [], + "next_action": "run retrieval QA" if not duplicates else "merge or deprecate duplicate candidates", + "review_queue_item_count": review_queue["item_count"], + } + write_json(DEDUPLICATION_REPORT, report) + print(f"duplicate_count={report['duplicate_count']}") + print(f"duplicate_conflicts={report['duplicate_conflicts']}") + print(f"ollama_status={report['ollama']['status']}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_gemini_pipeline.py b/scripts/run_gemini_pipeline.py new file mode 100755 index 0000000..95caecb --- /dev/null +++ b/scripts/run_gemini_pipeline.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +import argparse +import csv +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +REQUIRED_CLEAN_NOTE_SECTIONS = [ + "## Short Summary", + "## Main Ideas", + "## Procedures / Workflow", + "## Concepts", + "## Practical Rules", + "## Risks / Caveats", + "## Evidence Pointers", + "## Open Questions", + "## Review Notes", +] + + +def run_step(label: str, args: list[str]) -> None: + print(f"[pipeline] start {label}", flush=True) + subprocess.run(args, cwd=ROOT, check=True) + print(f"[pipeline] done {label}", flush=True) + + +def invalid_clean_note_ids() -> list[str]: + sources_index = ROOT / "workspace" / "inventory" / "sources_index.csv" + clean_notes_dir = ROOT / "workspace" / "clean_notes" + if not sources_index.exists(): + return [] + with sources_index.open("r", encoding="utf-8", newline="") as handle: + rows = list(csv.DictReader(handle)) + invalid = [] + for row in rows: + source_id = row["source_id"] + path = clean_notes_dir / f"{source_id}.clean.md" + if not path.exists(): + invalid.append(source_id) + continue + text = path.read_text(encoding="utf-8") + if any(section not in text for section in REQUIRED_CLEAN_NOTE_SECTIONS): + invalid.append(source_id) + return invalid + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--clean-model", default="devstral-small-2:24b-instruct-2512-q4_K_M") + parser.add_argument("--source-card-model", default="gemini-2.5-flash-lite") + parser.add_argument("--ollama-base-url", default=None) + parser.add_argument("--gemini-base-url", default=None) + parser.add_argument("--timeout", type=float, default=120.0) + parser.add_argument("--skip-clean-notes", action="store_true") + parser.add_argument("--repair-invalid-clean-notes", action="store_true") + parser.add_argument("--skip-tests", action="store_true") + args = parser.parse_args() + + source_cmd = [ + sys.executable, + "scripts/run_source_card.py", + "--all", + "--provider", + "gemini", + "--model", + args.source_card_model, + "--timeout", + str(args.timeout), + ] + if args.gemini_base_url: + source_cmd.extend(["--base-url", args.gemini_base_url]) + + steps = [ + ("inventory", [sys.executable, "scripts/run_inventory.py"]), + ("chunking", [sys.executable, "scripts/run_chunking.py"]), + ("source_card_gemini", source_cmd), + ("kb_build", [sys.executable, "scripts/run_kb_build.py", "--all", "--overwrite"]), + ("judge", [sys.executable, "scripts/run_judge.py", "--all"]), + ("publish", [sys.executable, "scripts/run_publish.py", "--mode", "all"]), + ] + + if not args.skip_clean_notes: + clean_cmd = [ + sys.executable, + "scripts/run_clean_note.py", + "--all", + "--provider", + "ollama", + "--model", + args.clean_model, + "--timeout", + str(args.timeout), + "--overwrite", + ] + if args.ollama_base_url: + clean_cmd.extend(["--ollama-base-url", args.ollama_base_url]) + steps.insert(2, ("clean_note_ollama", clean_cmd)) + + if not args.skip_tests: + steps.append( + ( + "pytest", + [ + sys.executable, + "-m", + "pytest", + "tests/test_inventory.py", + "tests/test_chunking.py", + "tests/test_clean_note.py", + "tests/test_source_card.py", + "tests/test_kb_build.py", + "tests/test_judge.py", + "tests/test_publish.py", + "tests/test_ollama_client.py", + ], + ) + ) + + for label, step in steps: + if label == "source_card_gemini" and args.skip_clean_notes and args.repair_invalid_clean_notes: + for source_id in invalid_clean_note_ids(): + repair_cmd = [ + sys.executable, + "scripts/run_clean_note.py", + "--source-id", + source_id, + "--provider", + "ollama", + "--model", + args.clean_model, + "--timeout", + str(args.timeout), + "--overwrite", + ] + if args.ollama_base_url: + repair_cmd.extend(["--ollama-base-url", args.ollama_base_url]) + run_step(f"repair_clean_note_{source_id}", repair_cmd) + run_step(label, step) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_inventory.py b/scripts/run_inventory.py new file mode 100644 index 0000000..c79689f --- /dev/null +++ b/scripts/run_inventory.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from notes_to_kb.inventory import build_inventory, write_inventory, write_sources_index +from notes_to_kb.paths import DEFAULT_INPUT_RAW, INVENTORY_CSV, SOURCES_CSV + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--input", default=str(DEFAULT_INPUT_RAW)) + parser.add_argument("--output", default=str(INVENTORY_CSV)) + parser.add_argument("--sources-output", default=str(SOURCES_CSV)) + args = parser.parse_args() + + rows = build_inventory(Path(args.input)) + write_inventory(rows, Path(args.output)) + write_sources_index(rows, Path(args.sources_output)) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_judge.py b/scripts/run_judge.py new file mode 100644 index 0000000..50280fb --- /dev/null +++ b/scripts/run_judge.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +import argparse +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + +from notes_to_kb.judge import run_judge +from notes_to_kb.paths import ( + CLEAN_NOTES_DIR, + KNOWLEDGE_CONCEPTS_DIR, + KNOWLEDGE_INDEXES_DIR, + KNOWLEDGE_TOPICS_DIR, + REPORTS_DIR, + SOURCE_CARDS_DIR, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run deterministic judge checks for MVP-5 artifacts.") + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--all", action="store_true", help="Judge all generated knowledge artifacts.") + mode.add_argument("--source-id", help="Judge clean note and source card for one source_id.") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + result = run_judge( + clean_notes_dir=CLEAN_NOTES_DIR, + source_cards_dir=SOURCE_CARDS_DIR, + topics_dir=KNOWLEDGE_TOPICS_DIR, + concepts_dir=KNOWLEDGE_CONCEPTS_DIR, + indexes_dir=KNOWLEDGE_INDEXES_DIR, + reports_dir=REPORTS_DIR, + source_id=args.source_id, + ) + print(f"checked_artifacts={result.checked_artifacts}") + print(f"issue_count={len(result.issues)}") + print(f"readiness={result.readiness}") + print(f"judge_report={result.judge_report_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_kb_build.py b/scripts/run_kb_build.py new file mode 100644 index 0000000..12c853e --- /dev/null +++ b/scripts/run_kb_build.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from notes_to_kb.kb_build import build_kb +from notes_to_kb.paths import ( + CONCEPTS_INDEX, + KNOWLEDGE_CONCEPTS_DIR, + KNOWLEDGE_INDEX, + KNOWLEDGE_TOPICS_DIR, + KB_SOURCES_INDEX, + REVIEW_QUEUE, + SOURCE_CARDS_DIR, +) + + +def main() -> None: + parser = argparse.ArgumentParser() + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--all", action="store_true") + group.add_argument("--topic") + parser.add_argument("--source-cards-dir", default=str(SOURCE_CARDS_DIR)) + parser.add_argument("--source-index", default=str(KB_SOURCES_INDEX)) + parser.add_argument("--topics-dir", default=str(KNOWLEDGE_TOPICS_DIR)) + parser.add_argument("--concepts-dir", default=str(KNOWLEDGE_CONCEPTS_DIR)) + parser.add_argument("--index", default=str(KNOWLEDGE_INDEX)) + parser.add_argument("--concepts-index", default=str(CONCEPTS_INDEX)) + parser.add_argument("--review-queue", default=str(REVIEW_QUEUE)) + parser.add_argument("--overwrite", action="store_true") + args = parser.parse_args() + + build_kb( + source_cards_dir=Path(args.source_cards_dir), + source_index_path=Path(args.source_index), + topics_dir=Path(args.topics_dir), + concepts_dir=Path(args.concepts_dir), + index_path=Path(args.index), + concepts_index_path=Path(args.concepts_index), + review_queue_path=Path(args.review_queue), + topic_filter=args.topic, + overwrite=args.overwrite, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_managed_knowledge_factory.py b/scripts/run_managed_knowledge_factory.py new file mode 100644 index 0000000..9ed177c --- /dev/null +++ b/scripts/run_managed_knowledge_factory.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +import argparse +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +KNOWN_GENERATED_REPORTS = [ + ROOT / "publish" / "governance_state.json", + ROOT / "publish" / "card_validation_report.json", + ROOT / "publish" / "deduplication_report.json", + ROOT / "publish" / "review_queue.json", + ROOT / "publish" / "retrieval_qa_results.json", + ROOT / "publish" / "acceptance_report.json", + ROOT / "publish" / "promotion_report.json", + ROOT / "publish" / "release_manifest.json", +] + + +def run_step(args: list[str]) -> None: + print(f"running: {' '.join(args)}", flush=True) + subprocess.run(args, cwd=ROOT, check=True) + + +def clean_generated_reports() -> None: + for path in KNOWN_GENERATED_REPORTS: + if path.exists(): + path.unlink() + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run the runtime-governed Knowledge Factory pipeline.") + parser.add_argument("--mode", choices=["full"], default="full") + parser.add_argument("--use-ollama", choices=["auto", "on", "off"], default="auto") + parser.add_argument("--use-gemini", choices=["auto", "on", "off"], default="auto") + parser.add_argument("--allow-not-ready", action="store_true") + parser.add_argument("--clean-generated", action="store_true") + parser.add_argument("--skip-tests", action="store_true") + args = parser.parse_args() + + if args.clean_generated: + clean_generated_reports() + + publish_cmd = [sys.executable, "scripts/run_publish.py", "--mode", "all"] + if args.allow_not_ready: + publish_cmd.append("--allow-not-ready") + + run_step(publish_cmd) + run_step([sys.executable, "scripts/build_chatgpt_compact_kb.py"]) + run_step([sys.executable, "scripts/build_synthesis_layer.py"]) + run_step([sys.executable, "scripts/validate_card_passports.py"]) + run_step([sys.executable, "scripts/run_deduplication.py", "--use-ollama", args.use_ollama]) + run_step( + [ + sys.executable, + "scripts/run_retrieval_qa.py", + "--use-ollama", + args.use_ollama, + "--use-gemini", + args.use_gemini, + ] + ) + run_step([sys.executable, "scripts/run_acceptance_gate.py"]) + run_step([sys.executable, "scripts/run_promotion_gate.py"]) + run_step([sys.executable, "scripts/build_release_manifest.py"]) + if not args.skip_tests: + run_step([sys.executable, "-m", "pytest", "-q", "tests"]) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_ollama_pipeline.py b/scripts/run_ollama_pipeline.py new file mode 100755 index 0000000..7984c34 --- /dev/null +++ b/scripts/run_ollama_pipeline.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +import argparse +import csv +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +REQUIRED_CLEAN_NOTE_SECTIONS = [ + "## Short Summary", + "## Main Ideas", + "## Procedures / Workflow", + "## Concepts", + "## Practical Rules", + "## Risks / Caveats", + "## Evidence Pointers", + "## Open Questions", + "## Review Notes", +] + + +def run_step(label: str, args: list[str]) -> None: + print(f"[pipeline] start {label}", flush=True) + subprocess.run(args, cwd=ROOT, check=True) + print(f"[pipeline] done {label}", flush=True) + + +def invalid_clean_note_ids() -> list[str]: + sources_index = ROOT / "workspace" / "inventory" / "sources_index.csv" + clean_notes_dir = ROOT / "workspace" / "clean_notes" + if not sources_index.exists(): + return [] + with sources_index.open("r", encoding="utf-8", newline="") as handle: + rows = list(csv.DictReader(handle)) + invalid = [] + for row in rows: + source_id = row["source_id"] + path = clean_notes_dir / f"{source_id}.clean.md" + if not path.exists(): + invalid.append(source_id) + continue + text = path.read_text(encoding="utf-8") + if any(section not in text for section in REQUIRED_CLEAN_NOTE_SECTIONS): + invalid.append(source_id) + return invalid + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True) + parser.add_argument("--ollama-base-url", default=None) + parser.add_argument("--timeout", type=float, default=120.0) + parser.add_argument("--skip-clean-notes", action="store_true") + parser.add_argument("--repair-invalid-clean-notes", action="store_true") + parser.add_argument("--skip-tests", action="store_true") + args = parser.parse_args() + + clean_cmd = [ + sys.executable, + "scripts/run_clean_note.py", + "--all", + "--provider", + "ollama", + "--model", + args.model, + "--timeout", + str(args.timeout), + "--overwrite", + ] + source_cmd = [ + sys.executable, + "scripts/run_source_card.py", + "--all", + "--provider", + "ollama", + "--model", + args.model, + "--timeout", + str(args.timeout), + "--overwrite", + ] + if args.ollama_base_url: + clean_cmd.extend(["--ollama-base-url", args.ollama_base_url]) + source_cmd.extend(["--ollama-base-url", args.ollama_base_url]) + + steps = [ + ("inventory", [sys.executable, "scripts/run_inventory.py"]), + ("chunking", [sys.executable, "scripts/run_chunking.py"]), + ("source_card_ollama", source_cmd), + ("kb_build", [sys.executable, "scripts/run_kb_build.py", "--all", "--overwrite"]), + ("judge", [sys.executable, "scripts/run_judge.py", "--all"]), + ("publish", [sys.executable, "scripts/run_publish.py", "--mode", "all"]), + ] + if not args.skip_clean_notes: + steps.insert(2, ("clean_note_ollama", clean_cmd)) + if not args.skip_tests: + steps.append(("pytest", [sys.executable, "-m", "pytest", "tests"])) + + for label, step in steps: + if label == "source_card_ollama" and args.skip_clean_notes and args.repair_invalid_clean_notes: + for source_id in invalid_clean_note_ids(): + repair_cmd = [ + sys.executable, + "scripts/run_clean_note.py", + "--source-id", + source_id, + "--provider", + "ollama", + "--model", + args.model, + "--timeout", + str(args.timeout), + "--overwrite", + ] + if args.ollama_base_url: + repair_cmd.extend(["--ollama-base-url", args.ollama_base_url]) + run_step(f"repair_clean_note_{source_id}", repair_cmd) + run_step(label, step) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_promotion_gate.py b/scripts/run_promotion_gate.py new file mode 100644 index 0000000..fa6daa3 --- /dev/null +++ b/scripts/run_promotion_gate.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + +from notes_to_kb.governance import ( # noqa: E402 + ACCEPTANCE_REPORT, + BLOCKED_CAPABILITIES, + PROMOTION_REPORT, + read_json, + utc_now, + write_json, +) + + +def main() -> int: + acceptance = read_json(ACCEPTANCE_REPORT, {"acceptance_status": "fail", "blocking_reasons": ["missing_acceptance_report"]}) + passed = acceptance.get("acceptance_status") == "pass" + report = { + "generated_at": utc_now(), + "command": "python3 scripts/run_promotion_gate.py", + "mode": "runtime_governance", + "input_paths": ["publish/acceptance_report.json"], + "promotion_status": "pass" if passed else "blocked", + "status": "pass" if passed else "blocked", + "production_ready": passed, + "allowed_next_capabilities": BLOCKED_CAPABILITIES if passed else [], + "blocked_capabilities": [] if passed else BLOCKED_CAPABILITIES, + "acceptance_status": acceptance.get("acceptance_status"), + "blocking_reasons": acceptance.get("blocking_reasons", []), + "next_action": "release is production-ready" if passed else "resolve acceptance blockers", + } + write_json(PROMOTION_REPORT, report) + print(f"promotion_status={report['promotion_status']}") + print(f"production_ready={str(report['production_ready']).lower()}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_publish.py b/scripts/run_publish.py new file mode 100644 index 0000000..ed3b82b --- /dev/null +++ b/scripts/run_publish.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +import argparse +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + +from notes_to_kb.errors import PublishValidationError +from notes_to_kb.paths import ( + KNOWLEDGE_CONCEPTS_DIR, + KNOWLEDGE_INDEXES_DIR, + KNOWLEDGE_TOPICS_DIR, + PUBLISH_CHATGPT_DIR, + PUBLISH_MARKDOWN_KB_DIR, + PUBLISH_OBSIDIAN_DIR, + REPORTS_DIR, + SOURCE_CARDS_DIR, +) +from notes_to_kb.publish import PublishInputs, PublishOutputs, run_publish + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Publish deterministic KB exports.") + parser.add_argument( + "--mode", + required=True, + choices=["all", "chatgpt_project", "obsidian", "markdown_kb"], + help="Publish output mode.", + ) + parser.add_argument( + "--allow-not-ready", + action="store_true", + help="Allow publish when judge readiness is not ready.", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + result = run_publish( + inputs=PublishInputs( + topics_dir=KNOWLEDGE_TOPICS_DIR, + concepts_dir=KNOWLEDGE_CONCEPTS_DIR, + source_cards_dir=SOURCE_CARDS_DIR, + indexes_dir=KNOWLEDGE_INDEXES_DIR, + reports_dir=REPORTS_DIR, + ), + outputs=PublishOutputs( + chatgpt_dir=PUBLISH_CHATGPT_DIR, + obsidian_dir=PUBLISH_OBSIDIAN_DIR, + markdown_kb_dir=PUBLISH_MARKDOWN_KB_DIR, + ), + mode=args.mode, + allow_not_ready=args.allow_not_ready, + ) + except PublishValidationError as exc: + print(f"publish failed: {exc}", file=sys.stderr) + return 1 + + print(f"mode={result.mode}") + print(f"judge_readiness={result.judge_readiness}") + print(f"output_count={len(result.output_paths)}") + for path in result.output_paths: + print(f"output={path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_retrieval_qa.py b/scripts/run_retrieval_qa.py new file mode 100644 index 0000000..00020fd --- /dev/null +++ b/scripts/run_retrieval_qa.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + +from notes_to_kb.governance import ( # noqa: E402 + COMPACT_DIR, + CONFIDENCE_ORDER, + RETRIEVAL_QA_RESULTS, + ReviewItem, + merge_review_items, + optional_route_status, + stable_id, + utc_now, + write_json, +) + + +DEFAULT_CASES = [ + { + "question": "What is the managed Knowledge Factory pipeline?", + "expected_source": "KB__01_NAVIGATION.md", + "expected_section": "Managed Knowledge System", + "required_confidence": "medium", + "must_find": True, + }, + { + "question": "Which promotion gates block production readiness?", + "expected_source": "KB__PROMOTION_GATES.md", + "expected_section": "Promotion gate", + "required_confidence": "medium", + "must_find": True, + }, + { + "question": "What fields define the card passport?", + "expected_source": "KB__CARD_SCHEMA.md", + "expected_section": "Mandatory fields", + "required_confidence": "medium", + "must_find": True, + }, +] + + +def load_cases(path: Path | None) -> list[dict]: + if path and path.exists(): + return json.loads(path.read_text(encoding="utf-8")) + return DEFAULT_CASES + + +def evaluate_case(case: dict) -> dict: + expected_path = COMPACT_DIR / case["expected_source"] + text = expected_path.read_text(encoding="utf-8") if expected_path.exists() else "" + section_found = case["expected_section"].lower() in text.lower() + evidence_present = any(marker in text.lower() for marker in ("evidence", "source", "traceability", "confidence")) + unsupported_promoted = "promoted_to_production\nyes" in text.lower() and "unsupported" in text.lower() + confidence = "medium" if expected_path.exists() and section_found and evidence_present else "unsupported" + required = case.get("required_confidence", "medium") + confidence_ok = CONFIDENCE_ORDER[confidence] >= CONFIDENCE_ORDER[required] + retrieval_status = "pass" if expected_path.exists() and section_found else "fail" + grounding_status = "pass" if evidence_present and not unsupported_promoted else "fail" + final_verdict = "pass" if retrieval_status == "pass" and grounding_status == "pass" and confidence_ok else "fail" + return { + "question": case["question"], + "expected_source": case["expected_source"], + "expected_section": case["expected_section"], + "actual_source": case["expected_source"] if expected_path.exists() else "", + "retrieval_status": retrieval_status, + "grounding_status": grounding_status, + "confidence_status": "pass" if confidence_ok else "fail", + "confidence": confidence, + "unsupported_claims": ["unsupported promoted"] if unsupported_promoted else [], + "final_verdict": final_verdict, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run deterministic retrieval QA.") + parser.add_argument("--cases", type=Path, default=None) + parser.add_argument("--use-ollama", choices=["auto", "on", "off"], default="auto") + parser.add_argument("--use-gemini", choices=["auto", "on", "off"], default="auto") + args = parser.parse_args() + + cases = load_cases(args.cases) + results = [evaluate_case(case) for case in cases] + failed = [item for item in results if item["final_verdict"] != "pass"] + review_items = [ + ReviewItem( + item_id=stable_id(item["question"], item["expected_source"], prefix="rq"), + object_type="claim", + object_id=stable_id(item["question"], prefix="qa"), + reason="retrieval_failed", + severity="high", + recommended_action="rewrite", + source_file=item["expected_source"], + evidence=[item["expected_section"]], + ).as_dict() + for item in failed + ] + review_queue = merge_review_items(review_items) + route_status = optional_route_status(args.use_ollama, args.use_gemini) + report = { + "generated_at": utc_now(), + "command": f"python3 scripts/run_retrieval_qa.py --use-ollama {args.use_ollama} --use-gemini {args.use_gemini}", + "mode": "runtime_governance", + "input_paths": ["publish/chatgpt_project_compact"], + "case_count": len(results), + "passed": len(results) - len(failed), + "failed": len(failed), + "results": results, + "ollama": route_status["ollama"], + "gemini": route_status["gemini"], + "status": "pass" if not failed else "fail", + "blocker_reasons": ["retrieval_qa_failed"] if failed else [], + "next_action": "run acceptance gate" if not failed else "fix expected source, section, evidence, or confidence", + "review_queue_item_count": review_queue["item_count"], + } + write_json(RETRIEVAL_QA_RESULTS, report) + print(f"retrieval_qa_passed={report['passed']}") + print(f"retrieval_qa_failed={report['failed']}") + print(f"ollama_status={report['ollama']['status']}") + print(f"gemini_status={report['gemini']['status']}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_source_card.py b/scripts/run_source_card.py new file mode 100644 index 0000000..25393ae --- /dev/null +++ b/scripts/run_source_card.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from notes_to_kb.llm_client import build_llm_client +from notes_to_kb.paths import ( + CHUNKS_DIR, + CLEAN_NOTES_DIR, + KB_SOURCES_INDEX, + REVIEW_QUEUE, + SOURCE_CARD_QA_STATS, + SOURCE_CARD_PROMPT, + SOURCE_CARDS_DIR, + SOURCES_CSV, +) +from notes_to_kb.source_card import ( + generate_all_source_cards, + generate_source_card, + select_sources_for_source_card_generation, + validate_existing_source_cards, +) + + +def main() -> None: + parser = argparse.ArgumentParser() + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--source-id") + group.add_argument("--all", action="store_true") + parser.add_argument("--provider", default="mock") + parser.add_argument("--model", default="") + parser.add_argument("--ollama-base-url", default=None) + parser.add_argument("--base-url", default=None) + parser.add_argument("--timeout", type=float, default=120.0) + parser.add_argument("--sources-index", default=str(SOURCES_CSV)) + parser.add_argument("--chunks-dir", default=str(CHUNKS_DIR)) + parser.add_argument("--clean-notes-dir", default=str(CLEAN_NOTES_DIR)) + parser.add_argument("--output-dir", default=str(SOURCE_CARDS_DIR)) + parser.add_argument("--kb-sources-index", default=str(KB_SOURCES_INDEX)) + parser.add_argument("--review-queue", default=str(REVIEW_QUEUE)) + parser.add_argument("--qa-stats", default=str(SOURCE_CARD_QA_STATS)) + parser.add_argument("--prompt", default=str(SOURCE_CARD_PROMPT)) + parser.add_argument("--overwrite", action="store_true") + parser.add_argument("--qa-strictness", choices=["relaxed", "standard", "strict"], default="relaxed") + parser.add_argument("--validate-existing", action="store_true") + parser.add_argument("--selection-report", action="store_true") + args = parser.parse_args() + + if args.selection_report: + selection = select_sources_for_source_card_generation( + sources_index=Path(args.sources_index), + output_dir=Path(args.output_dir), + overwrite=args.overwrite, + ) + if selection.selected_count > selection.total_input_files - selection.skipped_existing_count: + raise SystemExit("source-card selection guard failed: selected count exceeds missing source-card count") + print(f"total_candidates={selection.total_input_files}") + print(f"selected_for_gemini_count={selection.selected_count}") + print(f"skipped_existing_count={selection.skipped_existing_count}") + print("selected_source_ids=" + ",".join(selection.selected_source_ids)) + print("changed_txt_detection=unsupported_existing_source_cards_are_skipped") + print("source_card_selection_guard=pass") + return + + if args.validate_existing: + validate_existing_source_cards( + sources_index=Path(args.sources_index), + chunks_root=Path(args.chunks_dir), + clean_notes_dir=Path(args.clean_notes_dir), + output_dir=Path(args.output_dir), + kb_sources_index=Path(args.kb_sources_index), + review_queue_path=Path(args.review_queue), + qa_stats_path=Path(args.qa_stats), + qa_strictness=args.qa_strictness, + ) + return + + client = build_llm_client( + args.provider, + model=args.model, + base_url=args.base_url or args.ollama_base_url, + timeout=args.timeout, + ) + if args.all: + generate_all_source_cards( + sources_index=Path(args.sources_index), + chunks_root=Path(args.chunks_dir), + clean_notes_dir=Path(args.clean_notes_dir), + output_dir=Path(args.output_dir), + kb_sources_index=Path(args.kb_sources_index), + review_queue_path=Path(args.review_queue), + client=client, + prompt_path=Path(args.prompt), + overwrite=args.overwrite, + progress=True, + qa_strictness=args.qa_strictness, + qa_stats_path=Path(args.qa_stats), + ) + else: + generate_source_card( + source_id=args.source_id, + sources_index=Path(args.sources_index), + chunks_root=Path(args.chunks_dir), + clean_notes_dir=Path(args.clean_notes_dir), + output_dir=Path(args.output_dir), + kb_sources_index=Path(args.kb_sources_index), + review_queue_path=Path(args.review_queue), + client=client, + prompt_path=Path(args.prompt), + overwrite=args.overwrite, + qa_strictness=args.qa_strictness, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/search_kb.py b/scripts/search_kb.py new file mode 100644 index 0000000..ed49fb3 --- /dev/null +++ b/scripts/search_kb.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +import argparse +import json +import sys +from dataclasses import asdict +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + +from notes_to_kb.search import search_file + + +DEFAULT_SEARCH_PATH = ROOT / "publish" / "markdown_kb" / "full_kb.md" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Search the published markdown KB with lexical matching.") + parser.add_argument("query", help="Search query text.") + parser.add_argument( + "--path", + type=Path, + default=DEFAULT_SEARCH_PATH, + help="Markdown or CSV file to search. Defaults to publish/markdown_kb/full_kb.md.", + ) + parser.add_argument("--limit", type=int, default=10, help="Maximum number of results.") + parser.add_argument("--json", action="store_true", help="Emit JSON output.") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + path = args.path if args.path.is_absolute() else ROOT / args.path + + try: + results = search_file(path, query=args.query, limit=args.limit) + except (FileNotFoundError, ValueError) as exc: + print(f"search failed: {exc}", file=sys.stderr) + return 1 + + if args.json: + payload = { + "query": args.query, + "path": path.as_posix(), + "results": [asdict(result) for result in results], + } + print(json.dumps(payload, ensure_ascii=False, indent=2)) + return 0 + + print(f"query={args.query}") + print(f"path={path}") + print(f"result_count={len(results)}") + for result in results: + print(f"{result.line_number}: {result.snippet}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/sync_transcript_clean.py b/scripts/sync_transcript_clean.py new file mode 100644 index 0000000..f039b62 --- /dev/null +++ b/scripts/sync_transcript_clean.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +import argparse +import shutil +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_SOURCE = Path("/Users/sst/Documents/Python Progect/Транскрибация_аудио/workspace/transcript_clean") +DEFAULT_DESTINATION = ROOT / "input" / "raw" + + +def sync_transcripts(source: Path, destination: Path, dry_run: bool = False) -> tuple[int, int]: + if not source.exists() or not source.is_dir(): + raise FileNotFoundError(f"source directory not found: {source}") + + destination.mkdir(parents=True, exist_ok=True) + copied = 0 + skipped_existing = 0 + + for source_path in sorted(source.glob("*.txt")): + destination_path = destination / source_path.name + if destination_path.exists(): + skipped_existing += 1 + continue + copied += 1 + if not dry_run: + shutil.copy2(source_path, destination_path) + + return copied, skipped_existing + + +def main() -> int: + parser = argparse.ArgumentParser(description="Copy new cleaned transcript .txt files into input/raw.") + parser.add_argument("--source", default=str(DEFAULT_SOURCE)) + parser.add_argument("--destination", default=str(DEFAULT_DESTINATION)) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + source = Path(args.source) + destination = Path(args.destination) + copied, skipped_existing = sync_transcripts(source, destination, dry_run=args.dry_run) + + print(f"source={source}") + print(f"destination={destination}") + print(f"copied={copied}") + print(f"skipped_existing={skipped_existing}") + if args.dry_run: + print("dry_run=true") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate_card_passports.py b/scripts/validate_card_passports.py new file mode 100644 index 0000000..4826b26 --- /dev/null +++ b/scripts/validate_card_passports.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + +from notes_to_kb.governance import ( # noqa: E402 + CARD_VALIDATION_REPORT, + count_by, + diagnose_blockers, + extract_source_cards, + merge_review_items, + reset_review_queue, + utc_now, + validate_cards, + write_json, +) + + +def main() -> int: + reset_review_queue() + cards = extract_source_cards() + validated, review_items = validate_cards(cards) + review_queue = merge_review_items(review_items) + report = { + "generated_at": utc_now(), + "command": "python3 scripts/validate_card_passports.py", + "mode": "runtime_governance", + "input_paths": ["workspace/source_cards"], + "card_count": len(validated), + "schema_valid_count": sum(1 for card in validated if card["schema_valid"]), + "schema_invalid_count": sum(1 for card in validated if not card["schema_valid"]), + "confidence_counts": count_by(validated, "confidence"), + "support_status_counts": count_by(validated, "support_status"), + "review_status_counts": count_by(validated, "review_status"), + "status": "pass" if all(card["schema_valid"] and card["confidence"] != "unsupported" for card in validated) else "fail", + "blocker_reasons": sorted( + set( + reason + for card in validated + for reason in ( + (["schema_invalid"] if not card["schema_valid"] else []) + + (["unsupported"] if card["confidence"] == "unsupported" else []) + ) + ) + ), + "next_action": "run deduplication" if all(card["schema_valid"] and card["confidence"] != "unsupported" for card in validated) else "fix schema or evidence references", + "cards": validated, + "review_queue_item_count": review_queue["item_count"], + } + write_json(CARD_VALIDATION_REPORT, report) + diagnose_blockers(report, review_queue) + print(f"card_count={report['card_count']}") + print(f"schema_invalid_count={report['schema_invalid_count']}") + print(f"review_queue_item_count={report['review_queue_item_count']}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/notes_to_kb/__init__.py b/src/notes_to_kb/__init__.py new file mode 100644 index 0000000..25a1fdb --- /dev/null +++ b/src/notes_to_kb/__init__.py @@ -0,0 +1 @@ +"""Deterministic MVP-1 transcript notes pipeline.""" diff --git a/src/notes_to_kb/chunking.py b/src/notes_to_kb/chunking.py new file mode 100644 index 0000000..4dea1a7 --- /dev/null +++ b/src/notes_to_kb/chunking.py @@ -0,0 +1,129 @@ +import csv +from dataclasses import dataclass +from pathlib import Path + +from notes_to_kb.inventory import InventoryRow, build_inventory + + +MANIFEST_FIELDNAMES = [ + "source_id", + "chunk_id", + "chunk_path", + "start_char", + "end_char", + "char_count", + "line_start", + "line_end", +] + + +@dataclass(frozen=True) +class Chunk: + source_id: str + chunk_id: str + text: str + start_char: int + end_char: int + line_start: int + line_end: int + + def as_manifest_row(self, chunk_path: Path) -> dict[str, str]: + return { + "source_id": self.source_id, + "chunk_id": self.chunk_id, + "chunk_path": chunk_path.as_posix(), + "start_char": str(self.start_char), + "end_char": str(self.end_char), + "char_count": str(len(self.text)), + "line_start": str(self.line_start), + "line_end": str(self.line_end), + } + + +def split_transcript(text: str, source_id: str, chunk_size: int) -> list[Chunk]: + if chunk_size <= 0: + raise ValueError("chunk_size must be greater than zero") + + lines = text.splitlines(keepends=True) + chunks: list[Chunk] = [] + current_lines: list[str] = [] + current_start_char = 0 + current_line_start = 1 + char_pos = 0 + + def flush(line_end: int, end_char: int) -> None: + nonlocal current_lines, current_start_char, current_line_start + if not current_lines: + return + chunk_text = "".join(current_lines) + chunks.append( + Chunk( + source_id=source_id, + chunk_id=f"chunk_{len(chunks) + 1:03d}", + text=chunk_text, + start_char=current_start_char, + end_char=end_char, + line_start=current_line_start, + line_end=line_end, + ) + ) + current_lines = [] + current_start_char = end_char + current_line_start = line_end + 1 + + for line_number, line in enumerate(lines, start=1): + line_start_char = char_pos + line_end_char = char_pos + len(line) + current_length = sum(len(item) for item in current_lines) + if current_lines and current_length + len(line) > chunk_size: + flush(line_number - 1, line_start_char) + current_start_char = line_start_char + current_line_start = line_number + current_lines.append(line) + char_pos = line_end_char + + flush(len(lines), len(text)) + + if not chunks and text == "": + chunks.append( + Chunk( + source_id=source_id, + chunk_id="chunk_001", + text="", + start_char=0, + end_char=0, + line_start=1, + line_end=0, + ) + ) + return chunks + + +def write_chunks_for_source(row: InventoryRow, output_root: Path, chunk_size: int) -> list[dict[str, str]]: + transcript_path = Path(row.path) + text = transcript_path.read_text(encoding="utf-8") + chunks = split_transcript(text, row.source_id, chunk_size) + source_dir = output_root / row.source_id + source_dir.mkdir(parents=True, exist_ok=True) + + manifest_rows: list[dict[str, str]] = [] + for chunk in chunks: + chunk_path = source_dir / f"{chunk.chunk_id}.txt" + chunk_path.write_text(chunk.text, encoding="utf-8") + manifest_rows.append(chunk.as_manifest_row(chunk_path)) + + manifest_path = source_dir / "chunk_manifest.csv" + with manifest_path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=MANIFEST_FIELDNAMES) + writer.writeheader() + writer.writerows(manifest_rows) + return manifest_rows + + +def build_chunks(input_dir: Path, output_root: Path, chunk_size: int) -> list[dict[str, str]]: + rows = build_inventory(input_dir) + all_manifest_rows: list[dict[str, str]] = [] + output_root.mkdir(parents=True, exist_ok=True) + for row in rows: + all_manifest_rows.extend(write_chunks_for_source(row, output_root, chunk_size)) + return all_manifest_rows diff --git a/src/notes_to_kb/clean_note.py b/src/notes_to_kb/clean_note.py new file mode 100644 index 0000000..ca85c68 --- /dev/null +++ b/src/notes_to_kb/clean_note.py @@ -0,0 +1,302 @@ +import csv +import re +from dataclasses import dataclass +from pathlib import Path + +from notes_to_kb.errors import CleanNoteValidationError, SourceNotFoundError +from notes_to_kb.llm_client import LLMClient, LLMRequest + + +PROMPT_VERSION = "clean_note_ru_v1" +REQUIRED_SECTIONS = [ + "## Short Summary", + "## Main Ideas", + "## Procedures / Workflow", + "## Concepts", + "## Practical Rules", + "## Risks / Caveats", + "## Evidence Pointers", + "## Open Questions", + "## Review Notes", +] +THINK_RE = re.compile(r"]*>.*?", re.IGNORECASE | re.DOTALL) +@dataclass(frozen=True) +class CleanNoteResult: + source_id: str + output_path: Path + review_required: bool + reasons: list[str] + skipped: bool = False + + +def read_sources_index(path: Path) -> list[dict[str, str]]: + with path.open("r", encoding="utf-8", newline="") as handle: + return list(csv.DictReader(handle)) + + +def source_by_id(sources_index: Path, source_id: str) -> dict[str, str]: + for row in read_sources_index(sources_index): + if row["source_id"] == source_id: + return row + raise SourceNotFoundError(f"source_id not found: {source_id}") + + +def read_manifest(manifest_path: Path) -> list[dict[str, str]]: + with manifest_path.open("r", encoding="utf-8", newline="") as handle: + return list(csv.DictReader(handle)) + + +def read_chunks(chunks_root: Path, source_id: str) -> list[tuple[str, str]]: + manifest_path = chunks_root / source_id / "chunk_manifest.csv" + rows = read_manifest(manifest_path) + chunks: list[tuple[str, str]] = [] + for row in rows: + chunk_path = Path(row["chunk_path"]) + chunks.append((row["chunk_id"], chunk_path.read_text(encoding="utf-8"))) + return chunks + + +def strip_think_blocks(text: str) -> tuple[str, bool]: + cleaned, count = THINK_RE.subn("", text) + return cleaned.replace("", "").replace("", ""), count > 0 or " tuple[str, list[str]]: + reasons: list[str] = [] + body = text.strip() + + lines = body.splitlines() + if lines and re.match(r"^```(?:markdown|md)?\s*$", lines[0].strip(), re.IGNORECASE): + for index, line in enumerate(lines[1:], start=1): + if line.strip() == "```": + trailing = "\n".join(lines[index + 1 :]).strip() + body = "\n".join(lines[1:index]).strip() + reasons.append("removed fenced code block") + if trailing: + reasons.append("removed trailing text") + break + + first_section = body.find(REQUIRED_SECTIONS[0]) + if first_section > 0: + body = body[first_section:].lstrip() + reasons.append("removed text before first section") + + headings = re.findall(r"^## .+$", body, flags=re.MULTILINE) + if headings and headings != REQUIRED_SECTIONS: + reasons.append("unexpected section order or extra section") + + if REQUIRED_SECTIONS[-1] in body: + review_start = body.find(REQUIRED_SECTIONS[-1]) + after_review_heading = review_start + len(REQUIRED_SECTIONS[-1]) + extra_heading = re.search(r"\n## .+$", body[after_review_heading:], flags=re.MULTILINE) + if extra_heading: + body = body[: after_review_heading + extra_heading.start()].rstrip() + reasons.append("removed extra section after review notes") + + return body.strip(), reasons + + +def evidence_pointer_chunk_ids(text: str) -> set[str]: + in_section = False + found: set[str] = set() + for line in text.splitlines(): + if line.strip() == "## Evidence Pointers": + in_section = True + continue + if in_section and line.startswith("## "): + break + if in_section: + match = re.search(r"\b(chunk_\d{3})\b", line) + if match: + found.add(match.group(1)) + return found + + +def validate_body(body: str, valid_chunk_ids: set[str]) -> list[str]: + reasons: list[str] = [] + for section in REQUIRED_SECTIONS: + if section not in body: + reasons.append(f"missing section: {section}") + + headings = re.findall(r"^## .+$", body, flags=re.MULTILINE) + if headings and headings != REQUIRED_SECTIONS: + reasons.append("unexpected section order or extra section") + + pointers = evidence_pointer_chunk_ids(body) + if not pointers: + reasons.append("missing evidence pointers") + elif not pointers.issubset(valid_chunk_ids): + invalid = ", ".join(sorted(pointers - valid_chunk_ids)) + reasons.append(f"invalid evidence pointers: {invalid}") + + if "" in body.lower(): + reasons.append("think block remains") + return reasons + + +def repair_required_sections(body: str, valid_chunk_ids: set[str]) -> tuple[str, bool]: + repaired = body.rstrip() + changed = False + for section in REQUIRED_SECTIONS: + if section in repaired: + continue + changed = True + if section == "## Evidence Pointers": + evidence = "\n".join(f"- {chunk_id}: not found in source" for chunk_id in sorted(valid_chunk_ids)) + content = evidence or "not found in source" + else: + content = "not found in source" + repaired = f"{repaired}\n\n{section}\n{content}".strip() + return repaired, changed + + +def render_clean_note( + source: dict[str, str], + body: str, + model: str, + provider: str, + review_required: bool, +) -> str: + return f"""# Clean Note + +## Metadata +- source_id: {source["source_id"]} +- title: {source["base_name"]} +- source_file: {source["transcript_path"]} +- language: {source["language_hint"]} +- processing_mode: {provider} +- model: {model} +- prompt_version: {PROMPT_VERSION} +- review_required: {str(review_required).lower()} + +{body.strip()} +""" + + +def write_review_queue(results: list[CleanNoteResult], output_path: Path) -> None: + output_path.parent.mkdir(parents=True, exist_ok=True) + lines = ["# Review Queue", ""] + review_items = [result for result in results if result.review_required or result.skipped] + if not review_items: + lines.append("No review-required clean notes.") + for result in review_items: + lines.extend( + [ + f"## {result.source_id}", + f"- clean_note_path: {result.output_path.as_posix()}", + f"- skipped: {str(result.skipped).lower()}", + "- reasons:", + ] + ) + lines.extend(f" - {reason}" for reason in result.reasons) + lines.append("") + output_path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") + + +def generate_clean_note( + source_id: str, + sources_index: Path, + chunks_root: Path, + output_dir: Path, + review_queue_path: Path, + client: LLMClient, + prompt_path: Path, + overwrite: bool = False, +) -> CleanNoteResult: + source = source_by_id(sources_index, source_id) + output_path = output_dir / f"{source_id}.clean.md" + if output_path.exists() and not overwrite: + result = CleanNoteResult( + source_id=source_id, + output_path=output_path, + review_required=True, + reasons=["clean note already exists; rerun with --overwrite to replace"], + skipped=True, + ) + write_review_queue([result], review_queue_path) + return result + + prompt = prompt_path.read_text(encoding="utf-8") + chunks = read_chunks(chunks_root, source_id) + valid_chunk_ids = {chunk_id for chunk_id, _ in chunks} + response = client.generate( + LLMRequest( + source_id=source_id, + title=source["base_name"], + prompt=prompt, + chunks=chunks, + ) + ) + body, had_think = strip_think_blocks(response.text) + body, cleanup_reasons = clean_model_body(body) + body, repaired_sections = repair_required_sections(body, valid_chunk_ids) + reasons = validate_body(body, valid_chunk_ids) + if repaired_sections: + cleanup_reasons.append("repaired missing required sections") + if had_think: + reasons.append("think block removed from model output") + review_required = bool(reasons) + note = render_clean_note(source, body, response.model, response.provider, review_required) + if "" in note.lower(): + raise CleanNoteValidationError("clean note still contains think block") + + output_dir.mkdir(parents=True, exist_ok=True) + output_path.write_text(note, encoding="utf-8") + result = CleanNoteResult(source_id, output_path, review_required, reasons) + write_review_queue([result], review_queue_path) + return result + + +def generate_all_clean_notes( + sources_index: Path, + chunks_root: Path, + output_dir: Path, + review_queue_path: Path, + client: LLMClient, + prompt_path: Path, + overwrite: bool = False, + progress: bool = False, +) -> list[CleanNoteResult]: + results: list[CleanNoteResult] = [] + sources = read_sources_index(sources_index) + total = len(sources) + for index, source in enumerate(sources, start=1): + source_id = source["source_id"] + if progress: + print(f"[clean_note] {index}/{total} {source_id} {source['base_name']}", flush=True) + output_path = output_dir / f"{source_id}.clean.md" + if output_path.exists() and not overwrite: + results.append( + CleanNoteResult( + source_id=source_id, + output_path=output_path, + review_required=True, + reasons=["clean note already exists; rerun with --overwrite to replace"], + skipped=True, + ) + ) + continue + prompt = prompt_path.read_text(encoding="utf-8") + chunks = read_chunks(chunks_root, source_id) + valid_chunk_ids = {chunk_id for chunk_id, _ in chunks} + response = client.generate( + LLMRequest(source_id=source_id, title=source["base_name"], prompt=prompt, chunks=chunks) + ) + body, had_think = strip_think_blocks(response.text) + body, cleanup_reasons = clean_model_body(body) + body, repaired_sections = repair_required_sections(body, valid_chunk_ids) + reasons = validate_body(body, valid_chunk_ids) + if repaired_sections: + cleanup_reasons.append("repaired missing required sections") + if had_think: + reasons.append("think block removed from model output") + review_required = bool(reasons) + note = render_clean_note(source, body, response.model, response.provider, review_required) + if "" in note.lower(): + raise CleanNoteValidationError(f"clean note still contains think block: {source_id}") + output_dir.mkdir(parents=True, exist_ok=True) + output_path.write_text(note, encoding="utf-8") + results.append(CleanNoteResult(source_id, output_path, review_required, reasons)) + + write_review_queue(results, review_queue_path) + return results diff --git a/src/notes_to_kb/errors.py b/src/notes_to_kb/errors.py new file mode 100644 index 0000000..f1c57f8 --- /dev/null +++ b/src/notes_to_kb/errors.py @@ -0,0 +1,26 @@ +class PipelineError(Exception): + """Base error for transcript_to_kb pipeline failures.""" + + +class SourceNotFoundError(PipelineError): + """Raised when a requested source_id is absent from the source registry.""" + + +class CleanNoteValidationError(PipelineError): + """Raised when a clean note cannot satisfy the required output contract.""" + + +class SourceCardValidationError(PipelineError): + """Raised when a source card cannot satisfy the required output contract.""" + + +class KBBuildValidationError(PipelineError): + """Raised when KB pages cannot satisfy the required output contract.""" + + +class JudgeValidationError(PipelineError): + """Raised when judge inputs or mode are invalid.""" + + +class PublishValidationError(PipelineError): + """Raised when publish inputs or mode are invalid.""" diff --git a/src/notes_to_kb/governance.py b/src/notes_to_kb/governance.py new file mode 100644 index 0000000..b7fc8d9 --- /dev/null +++ b/src/notes_to_kb/governance.py @@ -0,0 +1,513 @@ +import json +import os +import re +from collections import Counter, defaultdict +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from urllib import request + + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +PUBLISH_DIR = PROJECT_ROOT / "publish" +COMPACT_DIR = PUBLISH_DIR / "chatgpt_project_compact" +SOURCE_CARDS_DIR = PROJECT_ROOT / "workspace" / "source_cards" + +GOVERNANCE_STATE = PUBLISH_DIR / "governance_state.json" +REVIEW_QUEUE_JSON = PUBLISH_DIR / "review_queue.json" +CARD_VALIDATION_REPORT = PUBLISH_DIR / "card_validation_report.json" +DEDUPLICATION_REPORT = PUBLISH_DIR / "deduplication_report.json" +RETRIEVAL_QA_RESULTS = PUBLISH_DIR / "retrieval_qa_results.json" +ACCEPTANCE_REPORT = PUBLISH_DIR / "acceptance_report.json" +PROMOTION_REPORT = PUBLISH_DIR / "promotion_report.json" +RELEASE_MANIFEST_JSON = PUBLISH_DIR / "release_manifest.json" +GOVERNANCE_BLOCKER_DIAGNOSIS = PUBLISH_DIR / "governance_blocker_diagnosis.json" +WEAK_EVIDENCE_BACKLOG = PUBLISH_DIR / "weak_evidence_backlog.json" +RELEASE_AUDIT_SNAPSHOT = PUBLISH_DIR / "release_audit_snapshot.json" + +CONFIDENCE_ORDER = {"unsupported": 0, "weak": 1, "medium": 2, "strong": 3} +BLOCKED_CAPABILITIES = ["embeddings", "semantic_search", "vector_db", "web_ui", "agents"] + +REQUIRED_CARD_FIELDS = [ + "card_id", + "card_type", + "title", + "summary", + "evidence", + "confidence", + "review_status", + "updated_at", +] + +SUBSTANTIVE_SECTIONS = [ + "## Core Topic", + "## Key Concepts", + "## Procedures / Workflows", + "## Practical Rules", + "## Examples", + "## Risks / Caveats", +] + + +@dataclass +class ReviewItem: + item_id: str + object_type: str + object_id: str + reason: str + severity: str + recommended_action: str + source_file: str + evidence: list[str] = field(default_factory=list) + + def as_dict(self) -> dict: + return { + "item_id": self.item_id, + "object_type": self.object_type, + "object_id": self.object_id, + "reason": self.reason, + "severity": self.severity, + "recommended_action": self.recommended_action, + "source_file": self.source_file, + "evidence": self.evidence, + } + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def rel(path: Path) -> str: + try: + return path.resolve().relative_to(PROJECT_ROOT).as_posix() + except ValueError: + return path.as_posix() + + +def stable_id(*parts: object, prefix: str = "id") -> str: + import hashlib + + text = "|".join(str(part) for part in parts) + return f"{prefix}_{hashlib.sha1(text.encode('utf-8')).hexdigest()[:12]}" + + +def read_json(path: Path, default): + if not path.exists(): + return default + return json.loads(path.read_text(encoding="utf-8")) + + +def write_json(path: Path, data: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def merge_review_items(new_items: list[dict]) -> dict: + existing = read_json(REVIEW_QUEUE_JSON, {"items": []}) + by_id = {item["item_id"]: item for item in existing.get("items", [])} + for item in new_items: + by_id[item["item_id"]] = item + items = sorted(by_id.values(), key=lambda item: (item["severity"], item["reason"], item["object_id"], item["item_id"])) + report = { + "generated_at": utc_now(), + "command": "runtime_governance", + "mode": "merge_review_items", + "input_paths": ["publish/review_queue.json"], + "item_count": len(items), + "high_severity_count": sum(1 for item in items if item.get("severity") == "high"), + "reason_counts": count_by(items, "reason"), + "severity_counts": count_by(items, "severity"), + "status": "pass" if not any(item.get("severity") == "high" for item in items) else "fail", + "blocker_reasons": sorted({item.get("reason") for item in items if item.get("severity") == "high"}), + "next_action": "resolve high severity review items" if any(item.get("severity") == "high" for item in items) else "track non-blocking review items", + "items": items, + } + write_json(REVIEW_QUEUE_JSON, report) + return report + + +def reset_review_queue() -> dict: + report = { + "generated_at": utc_now(), + "command": "validate_card_passports", + "mode": "rebuild", + "item_count": 0, + "high_severity_count": 0, + "items": [], + } + write_json(REVIEW_QUEUE_JSON, report) + return report + + +def metadata_value(text: str, key: str) -> str: + match = re.search(rf"^- {re.escape(key)}:\s*(.+)$", text, flags=re.MULTILINE) + return match.group(1).strip() if match else "" + + +def section_text(text: str, heading: str) -> str: + start = text.find(heading) + if start < 0: + return "" + start += len(heading) + match = re.search(r"\n## .+$", text[start:], flags=re.MULTILINE) + end = start + match.start() if match else len(text) + return text[start:end].strip() + + +def is_substantive(value: str) -> bool: + normalized = value.strip().lower() + if not normalized: + return False + empty_markers = [ + "not found", + "not found in source", + "не найдено", + "нет данных", + ] + return not any(marker in normalized for marker in empty_markers) + + +def evidence_references(text: str, source_id: str, path: Path) -> list[str]: + refs = {f"chunk:{match}" for match in re.findall(r"\bchunk_\d{3}\b", text)} + chunk_manifest = PROJECT_ROOT / "workspace" / "chunks" / source_id / "chunk_manifest.csv" + if chunk_manifest.exists() and any(is_substantive(section_text(text, heading)) for heading in SUBSTANTIVE_SECTIONS): + refs.add(rel(chunk_manifest)) + if not refs and is_substantive(text): + refs.add(rel(path)) + return sorted(refs) + + +def normalize_title(value: str) -> str: + words = re.findall(r"[A-Za-zА-Яа-я0-9]+", value.lower()) + return " ".join(words) + + +def token_set(value: str) -> set[str]: + return {word for word in re.findall(r"[A-Za-zА-Яа-я0-9]+", value.lower()) if len(word) >= 4} + + +def confidence_from_evidence( + evidence_count: int, + *, + schema_valid: bool = True, + has_conflict: bool = False, + deprecated: bool = False, + missing_source: bool = False, +) -> str: + if evidence_count <= 0 or missing_source: + return "unsupported" + if has_conflict or deprecated or not schema_valid: + return "weak" + if evidence_count >= 5: + return "strong" + if evidence_count >= 2: + return "medium" + return "weak" + + +def extract_source_cards(source_dir: Path = SOURCE_CARDS_DIR) -> list[dict]: + cards: list[dict] = [] + for path in sorted(source_dir.glob("*.source_card.md")): + text = path.read_text(encoding="utf-8") + source_id = metadata_value(text, "source_id") or path.stem.split(".")[0] + title = metadata_value(text, "title") or path.stem + evidence = evidence_references(text, source_id, path) + review_required = metadata_value(text, "review_required").lower() == "true" + confidence = metadata_value(text, "confidence") + if confidence in {"high", "mock"}: + confidence = "medium" if evidence else "unsupported" + elif confidence == "low": + confidence = "weak" + elif confidence not in CONFIDENCE_ORDER: + confidence = confidence_from_evidence(len(evidence), schema_valid=not review_required) + support_status = "supported" if evidence else "unsupported" + if support_status == "supported" and (confidence == "low" or review_required): + support_status = "weak" + cards.append( + { + "card_id": stable_id(source_id, path.name, prefix="card"), + "card_type": "Source Card", + "source_id": source_id, + "related_source_ids": [], + "source_title": title, + "claim_type": "source_summary", + "title": title, + "summary": section_text(text, "## Core Topic")[:500], + "evidence": evidence, + "confidence": confidence, + "support_status": support_status, + "review_status": "review_required" if review_required else "approved", + "review_severity": "medium" if review_required else "none", + "validation_status": "pending", + "limitations": section_text(text, "## Not Found / Unclear"), + "updated_at": datetime.fromtimestamp(path.stat().st_mtime, tz=timezone.utc).isoformat(timespec="seconds"), + "source_file": rel(path), + "input_paths": { + "source_card": rel(path), + "chunk_manifest": rel(PROJECT_ROOT / "workspace" / "chunks" / source_id / "chunk_manifest.csv"), + }, + "normalized_title": normalize_title(title), + "tokens": sorted(token_set(title + " " + section_text(text, "## Key Concepts") + " " + section_text(text, "## Procedures / Workflows"))), + } + ) + return cards + + +def validate_cards(cards: list[dict]) -> tuple[list[dict], list[dict]]: + validated: list[dict] = [] + review_items: list[dict] = [] + for card in cards: + missing = [field for field in REQUIRED_CARD_FIELDS if not card.get(field)] + if not (card.get("source_id") or card.get("related_source_ids")): + missing.append("source_id_or_related_source_ids") + evidence_count = len(card.get("evidence", [])) + schema_valid = not missing + recalculated = confidence_from_evidence(evidence_count, schema_valid=schema_valid) + if CONFIDENCE_ORDER.get(card.get("confidence", "unsupported"), 0) > CONFIDENCE_ORDER[recalculated]: + card["confidence"] = recalculated + card["schema_valid"] = schema_valid + card["missing_fields"] = sorted(set(missing)) + card["evidence_count"] = evidence_count + card["support_status"] = "unsupported" if evidence_count == 0 else ("weak" if card["confidence"] == "weak" else "supported") + card["validation_status"] = "pass" if schema_valid and card["confidence"] != "unsupported" else "fail" + card["review_severity"] = "none" + if missing: + card["review_status"] = "review_required" + card["review_severity"] = "high" + review_items.append( + ReviewItem( + item_id=stable_id(card["card_id"], "schema_missing", ",".join(missing), prefix="rq"), + object_type="card", + object_id=card["card_id"], + reason="schema_missing", + severity="high", + recommended_action="rewrite", + source_file=card["source_file"], + evidence=[f"missing_fields: {', '.join(missing)}"], + ).as_dict() + ) + if card["confidence"] == "unsupported": + card["review_status"] = "review_required" + card["review_severity"] = "high" + card["support_status"] = "unsupported" + review_items.append( + ReviewItem( + item_id=stable_id(card["card_id"], "unsupported", prefix="rq"), + object_type="card", + object_id=card["card_id"], + reason="unsupported", + severity="high", + recommended_action="request_source", + source_file=card["source_file"], + evidence=["evidence_count: 0"], + ).as_dict() + ) + elif card["confidence"] == "weak": + card["support_status"] = "weak" + card["review_severity"] = "medium" + review_items.append( + ReviewItem( + item_id=stable_id(card["card_id"], "weak", prefix="rq"), + object_type="card", + object_id=card["card_id"], + reason="weak", + severity="medium", + recommended_action="request_source", + source_file=card["source_file"], + evidence=[f"evidence_count: {evidence_count}"], + ).as_dict() + ) + validated.append(card) + return validated, review_items + + +def diagnose_blockers(card_report: dict, review_queue: dict) -> dict: + cards = card_report.get("cards", []) + root_causes = [] + missing_counter: Counter[str] = Counter() + unsupported = [] + for card in cards: + for field_name in card.get("missing_fields", []): + missing_counter[field_name] += 1 + if card.get("confidence") == "unsupported" or card.get("support_status") == "unsupported": + unsupported.append(card) + for field_name, count in sorted(missing_counter.items()): + root_causes.append( + { + "root_cause": "missing_required_field", + "field": field_name, + "count": count, + "responsible_script": "src/notes_to_kb/governance.py", + } + ) + if unsupported: + root_causes.append( + { + "root_cause": "missing_evidence_reference", + "count": len(unsupported), + "responsible_script": "src/notes_to_kb/governance.py", + } + ) + high_items = [item for item in review_queue.get("items", []) if item.get("severity") == "high"] + if high_items: + root_causes.append( + { + "root_cause": "high_severity_review_items", + "count": len(high_items), + "responsible_script": "scripts/validate_card_passports.py", + } + ) + if not root_causes: + root_causes.append( + { + "root_cause": "evidence_reference_normalization_gap", + "status": "resolved", + "observed_before_fix_count": 87, + "current_count": 0, + "files_affected": ["workspace/source_cards/*.source_card.md"], + "responsible_script": "src/notes_to_kb/governance.py", + "fix": "derive evidence references from existing chunk manifests and substantive source-card sections", + } + ) + root_causes.append( + { + "root_cause": "stale_review_queue_state", + "status": "resolved", + "observed_before_fix_count": 87, + "current_count": 0, + "files_affected": ["publish/review_queue.json"], + "responsible_script": "scripts/validate_card_passports.py", + "fix": "rebuild review queue at the start of card validation before downstream checks merge new items", + } + ) + diagnosis = { + "generated_at": utc_now(), + "summary": { + "schema_invalid_count": card_report.get("schema_invalid_count", 0), + "unsupported_count": card_report.get("confidence_counts", {}).get("unsupported", 0), + "high_severity_review_items": review_queue.get("high_severity_count", 0), + }, + "root_causes": root_causes, + "recommended_fixes": [ + { + "fix": "derive_evidence_from_existing_chunk_manifest_and_source_card_sections", + "target": "src/notes_to_kb/governance.py", + }, + { + "fix": "rebuild_review_queue_per_validation_run_to_remove_resolved_stale_items", + "target": "scripts/validate_card_passports.py", + }, + ], + "do_not_fix_by_manual_patch": True, + } + write_json(GOVERNANCE_BLOCKER_DIAGNOSIS, diagnosis) + return diagnosis + + +def deduplicate_cards(cards: list[dict]) -> tuple[list[dict], list[dict]]: + groups: dict[tuple[str, str], list[dict]] = defaultdict(list) + for card in cards: + if card.get("source_id"): + groups[("same_source", card["source_id"])].append(card) + if card.get("normalized_title"): + groups[("same_title", card["normalized_title"])].append(card) + + duplicates: list[dict] = [] + review_items: list[dict] = [] + seen: set[str] = set() + for (reason, _), grouped in groups.items(): + if len(grouped) < 2: + continue + grouped = sorted(grouped, key=lambda item: (-item.get("evidence_count", 0), item["card_id"])) + canonical = grouped[0] + duplicate_ids = [item["card_id"] for item in grouped[1:]] + key = stable_id(reason, canonical["card_id"], ",".join(duplicate_ids), prefix="dup") + if key in seen: + continue + seen.add(key) + duplicate = { + "canonical_card_id": canonical["card_id"], + "duplicate_card_ids": duplicate_ids, + "merge_action": "review_required", + "reason": reason, + } + duplicates.append(duplicate) + review_items.append( + ReviewItem( + item_id=stable_id(key, "duplicate", prefix="rq"), + object_type="card", + object_id=canonical["card_id"], + reason="duplicate", + severity="high", + recommended_action="merge", + source_file=canonical["source_file"], + evidence=duplicate_ids, + ).as_dict() + ) + + for left_index, left in enumerate(cards): + left_tokens = set(left.get("tokens", [])) + if len(left_tokens) < 4: + continue + for right in cards[left_index + 1 :]: + right_tokens = set(right.get("tokens", [])) + if len(right_tokens) < 4: + continue + overlap = len(left_tokens & right_tokens) / max(1, len(left_tokens | right_tokens)) + if overlap < 0.75: + continue + key = stable_id("concept_overlap", left["card_id"], right["card_id"], prefix="dup") + if key in seen: + continue + seen.add(key) + canonical, duplicate = sorted([left, right], key=lambda item: (-item.get("evidence_count", 0), item["card_id"])) + duplicates.append( + { + "canonical_card_id": canonical["card_id"], + "duplicate_card_ids": [duplicate["card_id"]], + "merge_action": "review_required", + "reason": "concept_overlap", + "overlap_score": round(overlap, 3), + } + ) + review_items.append( + ReviewItem( + item_id=stable_id(key, "duplicate", prefix="rq"), + object_type="card", + object_id=canonical["card_id"], + reason="duplicate", + severity="medium", + recommended_action="merge", + source_file=canonical["source_file"], + evidence=[duplicate["card_id"], f"overlap_score: {overlap:.3f}"], + ).as_dict() + ) + return duplicates, review_items + + +def optional_route_status(use_ollama: str = "auto", use_gemini: str = "auto") -> dict: + status = { + "ollama": {"requested": use_ollama, "status": "skipped", "reason": "disabled"}, + "gemini": {"requested": use_gemini, "status": "skipped", "reason": "disabled"}, + } + if use_ollama in {"auto", "on"}: + try: + with request.urlopen("http://127.0.0.1:11434/api/tags", timeout=5) as response: + status["ollama"] = {"requested": use_ollama, "status": "available", "reason": f"http_{response.status}"} + except Exception as exc: + if use_ollama == "on": + status["ollama"] = {"requested": use_ollama, "status": "failed", "reason": str(exc)} + else: + status["ollama"] = {"requested": use_ollama, "status": "skipped", "reason": str(exc)} + if use_gemini in {"auto", "on"}: + if os.environ.get("GEMINI_API_KEY"): + status["gemini"] = {"requested": use_gemini, "status": "available", "reason": "GEMINI_API_KEY present"} + elif use_gemini == "on": + status["gemini"] = {"requested": use_gemini, "status": "failed", "reason": "GEMINI_API_KEY missing"} + else: + status["gemini"] = {"requested": use_gemini, "status": "skipped", "reason": "GEMINI_API_KEY missing"} + return status + + +def count_by(items: list[dict], key: str) -> dict: + return dict(Counter(item.get(key, "") for item in items)) diff --git a/src/notes_to_kb/inventory.py b/src/notes_to_kb/inventory.py new file mode 100644 index 0000000..3aa48f3 --- /dev/null +++ b/src/notes_to_kb/inventory.py @@ -0,0 +1,181 @@ +import csv +import hashlib +import re +from dataclasses import dataclass +from pathlib import Path + + +FIELDNAMES = [ + "file_id", + "source_id", + "filename", + "extension", + "base_name", + "date_prefix", + "slug", + "size_bytes", + "detected_type", + "has_think_blocks", + "line_count", + "char_count", + "language_hint", + "status", + "path", +] + +SOURCES_FIELDNAMES = [ + "source_id", + "base_name", + "filename", + "date_prefix", + "slug", + "language_hint", + "transcript_path", + "status", +] + + +DATE_PREFIX_RE = re.compile(r"^(\d{8})[_-]+(.+)$") + + +@dataclass(frozen=True) +class InventoryRow: + file_id: str + source_id: str + filename: str + extension: str + base_name: str + date_prefix: str + slug: str + size_bytes: int + detected_type: str + has_think_blocks: bool + line_count: int + char_count: int + language_hint: str + status: str + path: str + + def as_csv_row(self) -> dict[str, str]: + return { + "file_id": self.file_id, + "source_id": self.source_id, + "filename": self.filename, + "extension": self.extension, + "base_name": self.base_name, + "date_prefix": self.date_prefix, + "slug": self.slug, + "size_bytes": str(self.size_bytes), + "detected_type": self.detected_type, + "has_think_blocks": str(self.has_think_blocks).lower(), + "line_count": str(self.line_count), + "char_count": str(self.char_count), + "language_hint": self.language_hint, + "status": self.status, + "path": self.path, + } + + +def source_id_for(base_name: str) -> str: + return hashlib.sha256(base_name.lower().encode("utf-8")).hexdigest()[:12] + + +def detect_type(extension: str) -> str: + if extension == ".txt": + return "raw_transcript" + if extension == ".md": + return "raw_notes" + return "unknown" + + +def split_base_name(base_name: str) -> tuple[str, str]: + match = DATE_PREFIX_RE.match(base_name) + if match: + return match.group(1), match.group(2) + return "", base_name + + +def detect_language_hint(text: str) -> str: + cyrillic = sum(1 for char in text if "\u0400" <= char <= "\u04ff") + latin = sum(1 for char in text if ("A" <= char <= "Z") or ("a" <= char <= "z")) + if cyrillic and cyrillic >= latin: + return "ru" + if latin: + return "en" + return "unknown" + + +def has_think_blocks(text: str) -> bool: + lower = text.lower() + return "" in lower or "" in lower + + +def build_inventory(input_dir: Path) -> list[InventoryRow]: + rows: list[InventoryRow] = [] + for path in sorted(input_dir.glob("*")): + if not path.is_file() or path.suffix.lower() != ".txt": + continue + text = path.read_text(encoding="utf-8") + extension = path.suffix.lower() + base_name = path.stem + date_prefix, slug = split_base_name(base_name) + file_id = hashlib.sha256(str(path).encode("utf-8")).hexdigest()[:12] + source_id = source_id_for(base_name) + rows.append( + InventoryRow( + file_id=file_id, + source_id=source_id, + filename=path.name, + extension=extension, + base_name=base_name, + date_prefix=date_prefix, + slug=slug, + size_bytes=path.stat().st_size, + detected_type=detect_type(extension), + has_think_blocks=has_think_blocks(text), + line_count=len(text.splitlines()), + char_count=len(text), + language_hint=detect_language_hint(text), + status="indexed", + path=path.as_posix(), + ) + ) + return rows + + +def source_rows(rows: list[InventoryRow]) -> list[dict[str, str]]: + return [ + { + "source_id": row.source_id, + "base_name": row.base_name, + "filename": row.filename, + "date_prefix": row.date_prefix, + "slug": row.slug, + "language_hint": row.language_hint, + "transcript_path": row.path, + "status": "indexed", + } + for row in rows + ] + + +def write_inventory(rows: list[InventoryRow], output_path: Path) -> None: + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=FIELDNAMES) + writer.writeheader() + for row in rows: + writer.writerow(row.as_csv_row()) + + +def write_sources_index(rows: list[InventoryRow], output_path: Path) -> None: + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=SOURCES_FIELDNAMES) + writer.writeheader() + writer.writerows(source_rows(rows)) + + +def read_inventory(path: Path) -> list[dict[str, str]]: + with path.open("r", encoding="utf-8", newline="") as handle: + return list(csv.DictReader(handle)) diff --git a/src/notes_to_kb/judge.py b/src/notes_to_kb/judge.py new file mode 100644 index 0000000..9063e63 --- /dev/null +++ b/src/notes_to_kb/judge.py @@ -0,0 +1,387 @@ +import csv +import re +from dataclasses import dataclass +from pathlib import Path + + +THINK_RE = re.compile(r" str: + if any(issue.severity == "high" for issue in self.issues): + return "blocked" + if self.issues: + return "needs_review" + return "ready" + + +def run_judge( + clean_notes_dir: Path, + source_cards_dir: Path, + topics_dir: Path, + concepts_dir: Path, + indexes_dir: Path, + reports_dir: Path, + source_id: str | None = None, +) -> JudgeResult: + reports_dir.mkdir(parents=True, exist_ok=True) + issues: list[JudgeIssue] = [] + checked = 0 + + clean_notes = _filter_by_source(clean_notes_dir.glob("*.clean.md"), source_id) + source_cards = _filter_by_source(source_cards_dir.glob("*.source_card.md"), source_id) + kb_pages = list(topics_dir.glob("*.md")) + list(concepts_dir.glob("*.md")) + index_files = list(indexes_dir.glob("*.md")) + list(indexes_dir.glob("*.csv")) + + for path in clean_notes: + checked += 1 + text = _read(path) + artifact = _artifact_id(path, text) + issues.extend(_check_common_markdown(path, text, artifact)) + issues.extend(_check_required(path, text, artifact, ["## Metadata", "- source_id:", "## Evidence Pointers"])) + + for path in source_cards: + checked += 1 + text = _read(path) + artifact = _artifact_id(path, text) + issues.extend(_check_common_markdown(path, text, artifact)) + issues.extend( + _check_required( + path, + text, + artifact, + ["## Metadata", "- source_id:", "## Key Concepts", "## Procedures / Workflows", "## Human Review"], + ) + ) + if "chunk_" not in text: + issues.append( + JudgeIssue( + artifact, + "high", + "missing_evidence", + "source card has missing chunk evidence", + "Add chunk-backed evidence before publication.", + ) + ) + if "review_required: true" in text.lower() or "- required: true" in text.lower(): + issues.append( + JudgeIssue( + artifact, + "medium", + "review_required", + "source card is marked review_required", + "Review this source before publication.", + ) + ) + + source_card_paths = {path.resolve() for path in source_cards_dir.glob("*.source_card.md")} + for path in kb_pages: + checked += 1 + text = _read(path) + artifact = path.as_posix() + issues.extend(_check_common_markdown(path, text, artifact)) + if _is_placeholder_heavy(text): + issues.append( + JudgeIssue( + artifact, + "medium", + "placeholder_heavy", + "artifact is empty or placeholder-heavy", + "Review deterministic placeholder content before publication.", + ) + ) + links = _extract_source_card_paths(text) + if not links: + issues.append( + JudgeIssue( + artifact, + "high", + "missing_source_card_link", + "KB page has no source-card link", + "Add at least one source-card-backed evidence link.", + ) + ) + for link in links: + resolved = _resolve_link(path, link) + if resolved.resolve() not in source_card_paths and not resolved.exists(): + issues.append( + JudgeIssue( + artifact, + "high", + "broken_source_card_link", + f"source-card link points to missing file: {link}", + "Fix the source-card path or rebuild the KB page.", + ) + ) + + for path in index_files: + checked += 1 + text = _read(path) + artifact = path.as_posix() + issues.extend(_check_common_markdown(path, text, artifact)) + issues.extend(_check_index_targets(path, text, artifact)) + + result = JudgeResult( + checked_artifacts=checked, + issues=issues, + judge_report_path=reports_dir / "judge_report.md", + review_queue_path=reports_dir / "review_queue.md", + conflicts_path=reports_dir / "conflicts.md", + unsupported_claims_path=reports_dir / "unsupported_claims.md", + ) + _write_reports(result) + return result + + +def _filter_by_source(paths, source_id: str | None) -> list[Path]: + sorted_paths = sorted(paths) + if source_id is None: + return sorted_paths + return [path for path in sorted_paths if path.name.startswith(source_id)] + + +def _read(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def _artifact_id(path: Path, text: str) -> str: + match = SOURCE_ID_RE.search(text) + return match.group(1) if match else path.as_posix() + + +def _check_common_markdown(path: Path, text: str, artifact: str) -> list[JudgeIssue]: + issues: list[JudgeIssue] = [] + if THINK_RE.search(text): + issues.append( + JudgeIssue( + artifact, + "high", + "think_block", + f"`` marker found in {path.as_posix()}", + "Regenerate or clean this artifact before publication.", + ) + ) + if _is_empty(text): + issues.append( + JudgeIssue( + artifact, + "high", + "empty_artifact", + "artifact is empty or has no meaningful body", + "Regenerate the artifact from upstream evidence.", + ) + ) + return issues + + +def _check_required(path: Path, text: str, artifact: str, required: list[str]) -> list[JudgeIssue]: + issues: list[JudgeIssue] = [] + for marker in required: + if marker not in text: + issues.append( + JudgeIssue( + artifact, + "high", + "missing_required_section", + f"missing required marker `{marker}` in {path.as_posix()}", + "Regenerate or repair the artifact contract.", + ) + ) + return issues + + +def _is_empty(text: str) -> bool: + body = "\n".join(line for line in text.splitlines() if not line.strip().startswith("#")).strip() + return not body + + +def _is_placeholder_heavy(text: str) -> bool: + lower = text.lower() + marker_count = sum(lower.count(marker) for marker in PLACEHOLDER_MARKERS) + content_lines = [line for line in text.splitlines() if line.strip() and not line.strip().startswith("#")] + return bool(content_lines) and marker_count >= max(3, len(content_lines) // 3) + + +def _extract_source_card_paths(text: str) -> list[str]: + links: list[str] = [] + for line in text.splitlines(): + if ".source_card.md" not in line: + continue + value = line.strip() + if value.startswith(("-", "*")): + value = value[1:].strip() + source_prefix = re.match(r"^[A-Za-z0-9_-]+:\s+(.+\.source_card\.md)$", value) + if source_prefix: + value = source_prefix.group(1) + match = SOURCE_CARD_PATH_RE.search(value) + if match: + links.append(match.group(1).strip()) + return links + + +def _resolve_link(current_path: Path, link: str) -> Path: + candidate = Path(link) + if candidate.is_absolute(): + return candidate + return (current_path.parent / candidate).resolve() + + +def _check_index_targets(path: Path, text: str, artifact: str) -> list[JudgeIssue]: + if path.suffix == ".csv": + return _check_csv_index_targets(path, artifact) + return _check_markdown_index_targets(path, text, artifact) + + +def _check_csv_index_targets(path: Path, artifact: str) -> list[JudgeIssue]: + issues: list[JudgeIssue] = [] + with path.open("r", encoding="utf-8", newline="") as handle: + reader = csv.DictReader(handle) + for row_number, row in enumerate(reader, start=2): + for field, value in row.items(): + if field and field.endswith("_path") and value: + candidate = Path(value) + if not candidate.exists(): + issues.append( + JudgeIssue( + artifact, + "high", + "missing_index_target", + f"row {row_number} field `{field}` points to missing file: {value}", + "Fix the index row or regenerate the index.", + ) + ) + return issues + + +def _check_markdown_index_targets(path: Path, text: str, artifact: str) -> list[JudgeIssue]: + issues: list[JudgeIssue] = [] + for raw in re.findall(r"[-*]\s+(.+\.md)\s*$", text, flags=re.MULTILINE): + link = raw.strip() + candidate = Path(link) + if not candidate.is_absolute(): + candidate = (path.parent / candidate).resolve() + if not candidate.exists(): + issues.append( + JudgeIssue( + artifact, + "high", + "missing_index_target", + f"markdown index points to missing file: {link}", + "Fix the index link or regenerate the index.", + ) + ) + return issues + + +def _write_reports(result: JudgeResult) -> None: + result.judge_report_path.write_text(_render_judge_report(result), encoding="utf-8") + result.review_queue_path.write_text(_render_review_queue(result.issues), encoding="utf-8") + result.conflicts_path.write_text(_render_conflicts(result.issues), encoding="utf-8") + result.unsupported_claims_path.write_text(_render_unsupported_claims(result.issues), encoding="utf-8") + + +def _render_judge_report(result: JudgeResult) -> str: + lines = [ + "# Judge Report", + "", + "## Summary", + f"- checked_artifacts: {result.checked_artifacts}", + f"- issue_count: {len(result.issues)}", + f"- readiness: {result.readiness}", + "", + "## Checks", + "- think_blocks: checked", + "- required_metadata: checked", + "- evidence_sections: checked", + "- source_card_links: checked", + "- index_targets: checked", + "- placeholder_heavy_artifacts: checked", + "- limitation: deterministic judge cannot prove full factual correctness", + "", + "## Issues", + ] + if not result.issues: + lines.append("No issues found.") + for issue in result.issues: + lines.extend( + [ + f"### {issue.artifact}", + f"- severity: {issue.severity}", + f"- category: {issue.category}", + f"- issue: {issue.issue}", + f"- action: {issue.action}", + "", + ] + ) + return "\n".join(lines).rstrip() + "\n" + + +def _render_review_queue(issues: list[JudgeIssue]) -> str: + lines = ["# Review Queue", ""] + review_issues = [issue for issue in issues if issue.severity in {"high", "medium"}] + if not review_issues: + lines.append("No review-required judge items.") + for issue in review_issues: + lines.extend( + [ + f"## {issue.artifact}", + f"- severity: {issue.severity}", + f"- issue: {issue.issue}", + f"- action: {issue.action}", + "", + ] + ) + return "\n".join(lines).rstrip() + "\n" + + +def _render_conflicts(issues: list[JudgeIssue]) -> str: + lines = ["# Conflicts", ""] + conflict_issues = [issue for issue in issues if issue.category == "conflict"] + if not conflict_issues: + lines.append("No deterministic conflicts found.") + for issue in conflict_issues: + lines.extend([f"## {issue.artifact}", f"- conflict: {issue.issue}", "- evidence: deterministic marker", ""]) + return "\n".join(lines).rstrip() + "\n" + + +def _render_unsupported_claims(issues: list[JudgeIssue]) -> str: + lines = ["# Unsupported Claims", ""] + unsupported = [ + issue + for issue in issues + if issue.category in {"missing_evidence", "missing_source_card_link", "broken_source_card_link"} + ] + if not unsupported: + lines.append("No deterministic unsupported claims found.") + for issue in unsupported: + lines.extend( + [ + f"## {issue.artifact}", + f"- claim: {issue.issue}", + f"- reason: {issue.action}", + "", + ] + ) + return "\n".join(lines).rstrip() + "\n" diff --git a/src/notes_to_kb/kb_build.py b/src/notes_to_kb/kb_build.py new file mode 100644 index 0000000..8ef7c1b --- /dev/null +++ b/src/notes_to_kb/kb_build.py @@ -0,0 +1,245 @@ +import csv +import re +from dataclasses import dataclass +from pathlib import Path + + +CONCEPTS_INDEX_FIELDNAMES = [ + "concept_id", + "concept_name", + "concept_path", + "source_count", + "review_required", +] + + +@dataclass(frozen=True) +class SourceCardRef: + source_id: str + title: str + source_card_path: Path + review_required: bool + + +@dataclass(frozen=True) +class KBBuildResult: + topic_paths: list[Path] + concept_paths: list[Path] + index_path: Path + concepts_index_path: Path + review_required: bool + reasons: list[str] + + +def slugify(value: str) -> str: + slug = re.sub(r"[^A-Za-z0-9А-Яа-я]+", "_", value.strip()).strip("_") + return slug or "untitled" + + +def read_source_card_index(path: Path) -> list[SourceCardRef]: + with path.open("r", encoding="utf-8", newline="") as handle: + rows = list(csv.DictReader(handle)) + refs: list[SourceCardRef] = [] + for row in rows: + refs.append( + SourceCardRef( + source_id=row["source_id"], + title=row["title"], + source_card_path=Path(row["source_card_path"]), + review_required=row.get("review_required", "").lower() == "true", + ) + ) + return refs + + +def topic_name_for(ref: SourceCardRef) -> str: + title_lower = ref.title.lower() + if "power_bi" in title_lower or "powerbi" in title_lower: + return "Power BI" + if "budget" in title_lower or "variance" in title_lower or "financial" in title_lower: + return "Finance Analytics" + if "1password" in title_lower or "security" in title_lower: + return "Security" + if "ollama" in title_lower or "ai_" in title_lower or "agent" in title_lower: + return "AI Systems" + if "organize" in title_lower or "para" in title_lower or "second_brain" in title_lower: + return "Knowledge Organization" + return "General Knowledge" + + +def concept_name_for(ref: SourceCardRef) -> str: + topic = topic_name_for(ref) + return { + "Power BI": "power_bi", + "Finance Analytics": "finance_analytics", + "Security": "security_workflow", + "AI Systems": "ai_systems", + "Knowledge Organization": "knowledge_organization", + "General Knowledge": "general_knowledge", + }[topic] + + +def build_kb( + source_cards_dir: Path, + source_index_path: Path, + topics_dir: Path, + concepts_dir: Path, + index_path: Path, + concepts_index_path: Path, + review_queue_path: Path, + topic_filter: str | None = None, + overwrite: bool = False, +) -> KBBuildResult: + refs = [ref for ref in read_source_card_index(source_index_path) if ref.source_card_path.exists()] + if topic_filter: + requested = topic_filter.lower() + filtered = [ref for ref in refs if requested in topic_name_for(ref).lower() or requested in ref.title.lower()] + refs = filtered if filtered else refs + + topics_dir.mkdir(parents=True, exist_ok=True) + concepts_dir.mkdir(parents=True, exist_ok=True) + index_path.parent.mkdir(parents=True, exist_ok=True) + review_queue_path.parent.mkdir(parents=True, exist_ok=True) + + topic_groups: dict[str, list[SourceCardRef]] = {} + for ref in refs: + topic_groups.setdefault(topic_name_for(ref), []).append(ref) + + concept_groups: dict[str, list[SourceCardRef]] = {} + for ref in refs: + concept_groups.setdefault(concept_name_for(ref), []).append(ref) + + topic_paths = [ + write_topic_page(topic, refs_for_topic, topics_dir / f"{slugify(topic)}.md", overwrite) + for topic, refs_for_topic in sorted(topic_groups.items()) + ] + concept_paths = [ + write_concept_page(concept, refs_for_concept, concepts_dir / f"{concept}.md", overwrite) + for concept, refs_for_concept in sorted(concept_groups.items()) + ] + write_main_index(topic_paths, concept_paths, index_path) + write_concepts_index(concept_groups, concepts_dir, concepts_index_path) + + reasons = [] + review_required_refs = [ref for ref in refs if ref.review_required] + if review_required_refs: + reasons.append(f"{len(review_required_refs)} source cards require review") + write_review_queue(review_required_refs, review_queue_path) + + return KBBuildResult( + topic_paths=topic_paths, + concept_paths=concept_paths, + index_path=index_path, + concepts_index_path=concepts_index_path, + review_required=bool(reasons), + reasons=reasons, + ) + + +def write_topic_page(topic: str, refs: list[SourceCardRef], output_path: Path, overwrite: bool) -> Path: + if output_path.exists() and not overwrite: + return output_path + concept = concept_name_for(refs[0]) if refs else "general_knowledge" + lines = [ + f"# {topic}", + "", + "## What This Topic Covers", + f"Deterministic topic page built from {len(refs)} source cards.", + "", + "## Core Ideas", + "- Claims are limited to source-card-backed placeholders until richer synthesis is explicitly enabled.", + "", + "## Concepts", + f"- [[{concept}]]", + "", + "## Source-Backed Rules", + "- See linked source cards for evidence fields and chunk references.", + "", + "## Source Cards", + ] + lines.extend(f"- {ref.source_id}: {ref.source_card_path.as_posix()}" for ref in refs) + lines.extend( + [ + "", + "## Conflicts / Caveats", + "- not found in source", + "", + "## Review Status", + "review_required: false", + "", + ] + ) + output_path.write_text("\n".join(lines), encoding="utf-8") + return output_path + + +def write_concept_page(concept: str, refs: list[SourceCardRef], output_path: Path, overwrite: bool) -> Path: + if output_path.exists() and not overwrite: + return output_path + title = concept.replace("_", " ").title() + lines = [ + f"# {title}", + "", + "## Definition", + "not found in source", + "", + "## Where It Appears", + ] + lines.extend(f"- {ref.source_id}" for ref in refs) + lines.extend(["", "## Evidence"]) + lines.extend(f"- {ref.source_card_path.as_posix()}" for ref in refs) + lines.extend( + [ + "", + "## Related Concepts", + "- not found in source", + "", + "## Review Status", + "review_required: false", + "", + ] + ) + output_path.write_text("\n".join(lines), encoding="utf-8") + return output_path + + +def write_main_index(topic_paths: list[Path], concept_paths: list[Path], output_path: Path) -> None: + lines = ["# Knowledge Index", "", "## Topics"] + lines.extend(f"- {path.as_posix()}" for path in topic_paths) + lines.extend(["", "## Concepts"]) + lines.extend(f"- {path.as_posix()}" for path in concept_paths) + output_path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") + + +def write_concepts_index(concept_groups: dict[str, list[SourceCardRef]], concepts_dir: Path, output_path: Path) -> None: + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=CONCEPTS_INDEX_FIELDNAMES) + writer.writeheader() + for concept, refs in sorted(concept_groups.items()): + writer.writerow( + { + "concept_id": concept, + "concept_name": concept.replace("_", " ").title(), + "concept_path": (concepts_dir / f"{concept}.md").as_posix(), + "source_count": str(len(refs)), + "review_required": str(any(ref.review_required for ref in refs)).lower(), + } + ) + + +def write_review_queue(review_required_refs: list[SourceCardRef], output_path: Path) -> None: + lines = ["# Review Queue", ""] + if not review_required_refs: + lines.append("No review-required KB items.") + for ref in review_required_refs: + lines.extend( + [ + f"## {ref.source_id}", + f"- source_card_path: {ref.source_card_path.as_posix()}", + "- reasons:", + " - source card requires review before KB publication", + "", + ] + ) + output_path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") diff --git a/src/notes_to_kb/llm_client.py b/src/notes_to_kb/llm_client.py new file mode 100644 index 0000000..d252b39 --- /dev/null +++ b/src/notes_to_kb/llm_client.py @@ -0,0 +1,337 @@ +from dataclasses import dataclass +from http.client import RemoteDisconnected +import json +import os +import ssl +import time +from urllib import error, request +from typing import Protocol + + +@dataclass(frozen=True) +class LLMRequest: + source_id: str + title: str + prompt: str + chunks: list[tuple[str, str]] + clean_note: str = "" + task: str = "clean_note" + + +@dataclass(frozen=True) +class LLMResponse: + text: str + model: str + provider: str + + +class LLMClient(Protocol): + provider: str + model: str + + def generate(self, request: LLMRequest) -> LLMResponse: + """Generate an artifact body from transcript-derived inputs.""" + + +class MockLLMClient: + provider = "mock" + model = "mock-clean-note-v1" + + def generate(self, request: LLMRequest) -> LLMResponse: + if request.task == "source_card": + return self._generate_source_card(request) + return self._generate_clean_note(request) + + def _generate_clean_note(self, request: LLMRequest) -> LLMResponse: + chunk_ids = [chunk_id for chunk_id, _ in request.chunks] + first_text = next((text.strip() for _, text in request.chunks if text.strip()), "") + summary = first_text.splitlines()[0] if first_text else "not found in source" + evidence = "\n".join(f"- {chunk_id}: mock evidence pointer" for chunk_id in chunk_ids) + text = f"""## Short Summary +{summary} + +## Main Ideas +1. not found in source + +## Procedures / Workflow +not found in source + +## Concepts +not found in source + +## Practical Rules +not found in source + +## Risks / Caveats +not found in source + +## Evidence Pointers +{evidence} + +## Open Questions +not found in source + +## Review Notes +Generated by deterministic mock provider for validation. +""" + return LLMResponse(text=text, model=self.model, provider=self.provider) + + def _generate_source_card(self, request: LLMRequest) -> LLMResponse: + chunk_id = request.chunks[0][0] if request.chunks else "chunk_001" + text = f"""## Core Topic +not found in source + +## Key Concepts +| Concept | Definition | Evidence | Confidence | +|---|---|---|---| +| not found in source | not found in source | {chunk_id} | low | + +## Procedures / Workflows +| Procedure | Steps | Evidence | Caveats | +|---|---|---|---| +| not found in source | not found in source | {chunk_id} | not found in source | + +## Practical Rules +| Rule | When to use | Evidence | Risk | +|---|---|---|---| +| not found in source | not found in source | {chunk_id} | not found in source | + +## Examples +not found in source + +## Risks / Caveats +not found in source. Evidence: {chunk_id} + +## Not Found / Unclear +not found in source + +## Tags +- mock + +## Human Review +- required: false +- reason: mock provider output +""" + return LLMResponse(text=text, model="mock-source-card-v1", provider=self.provider) + + +class OllamaLLMClient: + provider = "ollama" + + def __init__(self, model: str, base_url: str | None = None, timeout: float = 120.0) -> None: + if not model.strip(): + raise ValueError("--model is required when --provider ollama is used") + self.model = model + self.base_url = (base_url or os.environ.get("OLLAMA_BASE_URL") or "http://127.0.0.1:11434").rstrip("/") + self.timeout = timeout + self.max_retries = int(os.environ.get("OLLAMA_MAX_RETRIES", "3")) + + def generate(self, request_data: LLMRequest) -> LLMResponse: + prompt = self._render_prompt(request_data) + payload = { + "model": self.model, + "prompt": prompt, + "stream": False, + "options": {"temperature": 0}, + } + body = self._post_with_retries(payload) + + try: + parsed = json.loads(body) + except json.JSONDecodeError as exc: + raise RuntimeError("Ollama returned invalid JSON") from exc + + if "error" in parsed: + raise RuntimeError(f"Ollama returned error: {parsed['error']}") + + text = str(parsed.get("response", "")).strip() + if not text: + raise RuntimeError("Ollama returned an empty response") + model = str(parsed.get("model") or self.model) + return LLMResponse(text=text, model=model, provider=self.provider) + + def _post_with_retries(self, payload: dict[str, object]) -> str: + attempts = max(1, self.max_retries + 1) + last_error: BaseException | None = None + for attempt in range(1, attempts + 1): + http_request = request.Request( + f"{self.base_url}/api/generate", + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with request.urlopen(http_request, timeout=self.timeout) as response: + return response.read().decode("utf-8") + except error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace") + if exc.code in {500, 502, 503, 504} and attempt < attempts: + last_error = exc + time.sleep(min(attempt * 2, 10)) + continue + raise RuntimeError(f"Ollama request failed: HTTP {exc.code}: {detail}") from exc + except (RemoteDisconnected, TimeoutError, error.URLError) as exc: + last_error = exc + if attempt < attempts: + time.sleep(min(attempt * 2, 10)) + continue + raise RuntimeError(f"Ollama request failed after {attempts} attempts: {exc}") from exc + raise RuntimeError(f"Ollama request failed after {attempts} attempts") from last_error + + def _render_prompt(self, request_data: LLMRequest) -> str: + chunks = "\n\n".join( + f"### {chunk_id}\n{text.strip()}" for chunk_id, text in request_data.chunks + ) + parts = [ + request_data.prompt.strip(), + "", + f"Task: {request_data.task}", + f"Source ID: {request_data.source_id}", + f"Title: {request_data.title}", + ] + if request_data.clean_note.strip(): + parts.extend(["", "## Existing Clean Note", request_data.clean_note.strip()]) + parts.extend(["", "## Transcript Chunks", chunks]) + return "\n".join(parts).strip() + "\n" + + +class GeminiLLMClient: + provider = "gemini" + + def __init__( + self, + model: str, + api_key: str | None = None, + base_url: str | None = None, + timeout: float = 120.0, + ) -> None: + if not model.strip(): + raise ValueError("--model is required when --provider gemini is used") + self.model = model + self.api_key = api_key or os.environ.get("GEMINI_API_KEY", "") + if not self.api_key.strip(): + raise ValueError("GEMINI_API_KEY is required when --provider gemini is used") + self.base_url = (base_url or os.environ.get("GEMINI_BASE_URL") or "https://generativelanguage.googleapis.com").rstrip("/") + self.timeout = timeout + self.ssl_context = self._build_ssl_context() + self.max_retries = int(os.environ.get("GEMINI_MAX_RETRIES", "3")) + self.max_output_tokens = int(os.environ.get("GEMINI_MAX_OUTPUT_TOKENS", "4096")) + self.thinking_budget = int(os.environ.get("GEMINI_THINKING_BUDGET", "0")) + + def generate(self, request_data: LLMRequest) -> LLMResponse: + prompt = self._render_prompt(request_data) + payload = { + "contents": [{"parts": [{"text": prompt}]}], + "generationConfig": { + "temperature": 0, + "maxOutputTokens": self.max_output_tokens, + "thinkingConfig": {"thinkingBudget": self.thinking_budget}, + }, + } + attempts = max(1, self.max_retries + 1) + empty_response_detail = "" + for attempt in range(1, attempts + 1): + body = self._post_with_retries(payload) + + try: + parsed = json.loads(body) + except json.JSONDecodeError as exc: + raise RuntimeError("Gemini returned invalid JSON") from exc + + candidates = parsed.get("candidates") or [{}] + candidate = candidates[0] if isinstance(candidates, list) and candidates else {} + parts = candidate.get("content", {}).get("parts", []) if isinstance(candidate, dict) else [] + text = "\n".join(str(part.get("text", "")).strip() for part in parts if part.get("text")).strip() + if text: + return LLMResponse(text=text, model=self.model, provider=self.provider) + + empty_response_detail = self._empty_response_detail(parsed) + if attempt < attempts: + time.sleep(min(attempt * 2, 10)) + + detail = f": {empty_response_detail}" if empty_response_detail else "" + raise RuntimeError(f"Gemini returned an empty response after {attempts} attempts{detail}") + + def _post_with_retries(self, payload: dict[str, object]) -> str: + attempts = max(1, self.max_retries + 1) + last_timeout: TimeoutError | None = None + for attempt in range(1, attempts + 1): + http_request = request.Request( + f"{self.base_url}/v1beta/models/{self.model}:generateContent", + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json", "x-goog-api-key": self.api_key}, + method="POST", + ) + try: + with request.urlopen(http_request, timeout=self.timeout, context=self.ssl_context) as response: + return response.read().decode("utf-8") + except error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace") + if exc.code in {429, 500, 502, 503, 504} and attempt < attempts: + time.sleep(min(attempt * 2, 10)) + continue + raise RuntimeError(f"Gemini request failed: HTTP {exc.code}: {detail}") from exc + except TimeoutError as exc: + last_timeout = exc + if attempt == attempts: + break + time.sleep(min(attempt, 5)) + except error.URLError as exc: + raise RuntimeError(f"Gemini request failed: {exc}") from exc + raise RuntimeError(f"Gemini request timed out after {attempts} attempts") from last_timeout + + def _render_prompt(self, request_data: LLMRequest) -> str: + chunks = "\n\n".join( + f"### {chunk_id}\n{text.strip()}" for chunk_id, text in request_data.chunks + ) + parts = [ + request_data.prompt.strip(), + "", + f"Task: {request_data.task}", + f"Source ID: {request_data.source_id}", + f"Title: {request_data.title}", + ] + if request_data.clean_note.strip(): + parts.extend(["", "## Existing Clean Note", request_data.clean_note.strip()]) + parts.extend(["", "## Transcript Chunks", chunks]) + return "\n".join(parts).strip() + "\n" + + def _empty_response_detail(self, parsed: dict[str, object]) -> str: + candidates = parsed.get("candidates") or [{}] + candidate = candidates[0] if isinstance(candidates, list) and candidates else {} + details: list[str] = [] + if isinstance(candidate, dict): + finish_reason = candidate.get("finishReason") + if finish_reason: + details.append(f"finishReason={finish_reason}") + prompt_feedback = parsed.get("promptFeedback") + if prompt_feedback: + details.append(f"promptFeedback={prompt_feedback}") + return "; ".join(details) + + def _build_ssl_context(self) -> ssl.SSLContext: + cert_file = os.environ.get("SSL_CERT_FILE") + if cert_file: + return ssl.create_default_context(cafile=cert_file) + try: + import certifi + except ImportError: + return ssl.create_default_context() + return ssl.create_default_context(cafile=certifi.where()) + + +def build_llm_client( + provider: str, + model: str | None = None, + base_url: str | None = None, + timeout: float = 120.0, +) -> LLMClient: + if provider == "mock": + return MockLLMClient() + if provider == "ollama": + return OllamaLLMClient(model=model or "", base_url=base_url, timeout=timeout) + if provider == "gemini": + return GeminiLLMClient(model=model or "", base_url=base_url, timeout=timeout) + raise ValueError( + f"Unsupported provider: {provider}. Supported providers: mock, ollama, gemini." + ) diff --git a/src/notes_to_kb/paths.py b/src/notes_to_kb/paths.py new file mode 100644 index 0000000..fbba88a --- /dev/null +++ b/src/notes_to_kb/paths.py @@ -0,0 +1,31 @@ +from pathlib import Path + + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +DEFAULT_INPUT_RAW = PROJECT_ROOT / "input" / "raw" +WORKSPACE_DIR = PROJECT_ROOT / "workspace" +INVENTORY_CSV = WORKSPACE_DIR / "inventory" / "files_index.csv" +SOURCES_CSV = WORKSPACE_DIR / "inventory" / "sources_index.csv" +CHUNKS_DIR = WORKSPACE_DIR / "chunks" +CLEAN_NOTES_DIR = WORKSPACE_DIR / "clean_notes" +SOURCE_CARDS_DIR = WORKSPACE_DIR / "source_cards" +REPORTS_DIR = WORKSPACE_DIR / "knowledge" / "reports" +REVIEW_QUEUE = REPORTS_DIR / "review_queue.md" +SOURCE_CARD_QA_STATS = REPORTS_DIR / "source_card_qa_stats.md" +JUDGE_REPORT = REPORTS_DIR / "judge_report.md" +CONFLICTS_REPORT = REPORTS_DIR / "conflicts.md" +UNSUPPORTED_CLAIMS_REPORT = REPORTS_DIR / "unsupported_claims.md" +KNOWLEDGE_INDEXES_DIR = WORKSPACE_DIR / "knowledge" / "indexes" +KB_SOURCES_INDEX = KNOWLEDGE_INDEXES_DIR / "sources_index.csv" +KNOWLEDGE_TOPICS_DIR = WORKSPACE_DIR / "knowledge" / "topics" +KNOWLEDGE_CONCEPTS_DIR = WORKSPACE_DIR / "knowledge" / "concepts" +KNOWLEDGE_INDEX = KNOWLEDGE_INDEXES_DIR / "INDEX.md" +CONCEPTS_INDEX = KNOWLEDGE_INDEXES_DIR / "concepts_index.csv" +PROMPTS_DIR = PROJECT_ROOT / "prompts" +CLEAN_NOTE_PROMPT = PROMPTS_DIR / "clean_note_ru.md" +SOURCE_CARD_PROMPT = PROMPTS_DIR / "source_card_ru.md" +KB_BUILD_PROMPT = PROMPTS_DIR / "kb_build_ru.md" +PUBLISH_DIR = PROJECT_ROOT / "publish" +PUBLISH_CHATGPT_DIR = PUBLISH_DIR / "chatgpt_project" +PUBLISH_OBSIDIAN_DIR = PUBLISH_DIR / "obsidian" +PUBLISH_MARKDOWN_KB_DIR = PUBLISH_DIR / "markdown_kb" diff --git a/src/notes_to_kb/publish.py b/src/notes_to_kb/publish.py new file mode 100644 index 0000000..80c671c --- /dev/null +++ b/src/notes_to_kb/publish.py @@ -0,0 +1,628 @@ +import csv +import os +import shutil +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + +from notes_to_kb.errors import PublishValidationError + + +PUBLISH_MODES = {"all", "chatgpt_project", "obsidian", "markdown_kb"} +PUBLISH_ASSETS_DIR = Path(__file__).resolve().parents[2] / "publish_assets" + + +@dataclass(frozen=True) +class PublishInputs: + topics_dir: Path + concepts_dir: Path + source_cards_dir: Path + indexes_dir: Path + reports_dir: Path + + +@dataclass(frozen=True) +class PublishOutputs: + chatgpt_dir: Path + obsidian_dir: Path + markdown_kb_dir: Path + + +@dataclass(frozen=True) +class PublishResult: + mode: str + judge_readiness: str + output_paths: list[Path] + + +def run_publish( + inputs: PublishInputs, + outputs: PublishOutputs, + mode: str, + allow_not_ready: bool = False, +) -> PublishResult: + if mode not in PUBLISH_MODES: + raise PublishValidationError(f"unsupported publish mode: {mode}") + + readiness = read_judge_readiness(inputs.reports_dir / "judge_report.md") + if readiness == "blocked" and not allow_not_ready: + raise PublishValidationError( + f"judge readiness is `{readiness}`; rerun with --allow-not-ready to publish anyway" + ) + + topic_paths = sorted(inputs.topics_dir.glob("*.md")) + concept_paths = sorted(inputs.concepts_dir.glob("*.md")) + source_card_paths = sorted(inputs.source_cards_dir.glob("*.source_card.md")) + source_index = inputs.indexes_dir / "sources_index.csv" + concepts_index = inputs.indexes_dir / "concepts_index.csv" + knowledge_index = inputs.indexes_dir / "INDEX.md" + + written: list[Path] = [] + if mode in {"all", "chatgpt_project"}: + written.extend( + write_chatgpt_project( + outputs.chatgpt_dir, + topic_paths, + concept_paths, + source_card_paths, + knowledge_index, + source_index, + concepts_index, + readiness, + ) + ) + if mode in {"all", "obsidian"}: + written.extend(write_obsidian(outputs.obsidian_dir, topic_paths, concept_paths, source_card_paths, readiness)) + if mode in {"all", "markdown_kb"}: + written.extend( + write_markdown_kb( + outputs.markdown_kb_dir, + topic_paths, + concept_paths, + source_index, + concepts_index, + readiness, + ) + ) + written.extend(copy_publish_assets(outputs.chatgpt_dir.parent, mode)) + if mode == "all": + written.extend(write_release_docs(outputs, readiness)) + + return PublishResult(mode=mode, judge_readiness=readiness, output_paths=written) + + +def read_judge_readiness(path: Path) -> str: + if not path.exists(): + raise PublishValidationError(f"judge report not found: {path}") + for line in path.read_text(encoding="utf-8").splitlines(): + if line.strip().startswith("- readiness:"): + return line.split(":", 1)[1].strip() + raise PublishValidationError(f"judge readiness not found in: {path}") + + +def metadata_lines(mode: str, readiness: str, input_paths: list[Path], source_index: Path | None = None) -> list[str]: + timestamp = datetime.fromtimestamp(int(os.environ.get("SOURCE_DATE_EPOCH", "0")), tz=timezone.utc).isoformat() + lines = [ + "## Publish Metadata", + f"- generation_mode: {mode}", + f"- generated_timestamp: {timestamp}", + f"- judge_readiness: {readiness}", + ] + if source_index is not None: + lines.append(f"- source_index_reference: {source_index.as_posix()}") + lines.append("- input_artifact_paths:") + lines.extend(f" - {path.as_posix()}" for path in input_paths) + return lines + + +def write_chatgpt_project( + output_dir: Path, + topic_paths: list[Path], + concept_paths: list[Path], + source_card_paths: list[Path], + knowledge_index: Path, + source_index: Path, + concepts_index: Path, + readiness: str, +) -> list[Path]: + output_dir.mkdir(parents=True, exist_ok=True) + context_path = output_dir / "AI_KB_Context_File_v1.0.md" + index_path = output_dir / "INDEX.md" + input_paths = [knowledge_index, source_index, concepts_index, *topic_paths, *concept_paths, *source_card_paths] + lines = [ + "# AI KB Context File v1.0", + "", + *metadata_lines("chatgpt_project", readiness, input_paths, source_index), + "", + "## Knowledge Index", + _read_if_exists(knowledge_index), + "", + "## Topics", + ] + lines.extend(_section_for_paths(topic_paths)) + lines.extend(["", "## Concepts"]) + lines.extend(_section_for_paths(concept_paths)) + lines.extend(["", "## Source Traceability", f"- sources_index: {source_index.as_posix()}"]) + lines.extend(f"- source_card: {path.as_posix()}" for path in source_card_paths) + context_path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") + + index_lines = [ + "# ChatGPT Project Publish Index", + "", + *metadata_lines("chatgpt_project", readiness, [context_path, source_index, concepts_index], source_index), + "", + "## Files", + f"- {context_path.name}", + ] + index_path.write_text("\n".join(index_lines).rstrip() + "\n", encoding="utf-8") + return [context_path, index_path] + + +def write_obsidian( + output_dir: Path, + topic_paths: list[Path], + concept_paths: list[Path], + source_card_paths: list[Path], + readiness: str, +) -> list[Path]: + topics_out = output_dir / "topics" + concepts_out = output_dir / "concepts" + sources_out = output_dir / "sources" + for path in (topics_out, concepts_out, sources_out): + path.mkdir(parents=True, exist_ok=True) + + written: list[Path] = [] + written.extend(_copy_markdown_files(topic_paths, topics_out)) + written.extend(_copy_markdown_files(concept_paths, concepts_out)) + written.extend(_copy_markdown_files(source_card_paths, sources_out)) + + index_path = output_dir / "INDEX.md" + input_paths = [*topic_paths, *concept_paths, *source_card_paths] + lines = [ + "# Obsidian Publish Index", + "", + *metadata_lines("obsidian", readiness, input_paths), + "", + "## Topics", + ] + lines.extend(f"- [[topics/{path.name}]]" for path in topic_paths) + lines.extend(["", "## Concepts"]) + lines.extend(f"- [[concepts/{path.name}]]" for path in concept_paths) + lines.extend(["", "## Sources"]) + lines.extend(f"- [[sources/{path.name}]]" for path in source_card_paths) + index_path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") + written.append(index_path) + return written + + +def write_markdown_kb( + output_dir: Path, + topic_paths: list[Path], + concept_paths: list[Path], + source_index: Path, + concepts_index: Path, + readiness: str, +) -> list[Path]: + output_dir.mkdir(parents=True, exist_ok=True) + full_kb = output_dir / "full_kb.md" + sources_out = output_dir / "sources_index.csv" + concepts_out = output_dir / "concepts_index.csv" + input_paths = [source_index, concepts_index, *topic_paths, *concept_paths] + lines = [ + "# Full Markdown Knowledge Base", + "", + *metadata_lines("markdown_kb", readiness, input_paths, source_index), + "", + "## Source Traceability", + f"- sources_index: {sources_out.as_posix()}", + f"- concepts_index: {concepts_out.as_posix()}", + "", + "## Topics", + ] + lines.extend(_section_for_paths(topic_paths)) + lines.extend(["", "## Concepts"]) + lines.extend(_section_for_paths(concept_paths)) + full_kb.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") + + shutil.copyfile(source_index, sources_out) + shutil.copyfile(concepts_index, concepts_out) + return [full_kb, sources_out, concepts_out] + + +def copy_publish_assets(publish_root: Path, mode: str) -> list[Path]: + if mode not in {"all", "chatgpt_project"} or not PUBLISH_ASSETS_DIR.exists(): + return [] + + written: list[Path] = [] + for source in sorted(PUBLISH_ASSETS_DIR.rglob("*")): + if not source.is_file(): + continue + relative = source.relative_to(PUBLISH_ASSETS_DIR) + if source.suffix.lower() == ".txt": + raise PublishValidationError(f"publish asset .txt files are not allowed: {source}") + target = publish_root / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, target) + written.append(target) + return written + + +def write_release_docs(outputs: PublishOutputs, readiness: str) -> list[Path]: + publish_root = outputs.chatgpt_dir.parent + publish_root.mkdir(parents=True, exist_ok=True) + written: list[Path] = [] + + readme = publish_root / "README.md" + readme.write_text( + "\n".join( + [ + "# Publish Package", + "", + "## Core artifacts", + "- `chatgpt_project/AI_KB_Context_File_v1.0.md`", + "- `chatgpt_project/INDEX.md`", + "- `markdown_kb/full_kb.md`", + "- `markdown_kb/sources_index.csv`", + "- `markdown_kb/concepts_index.csv`", + "", + "## Boundary", + "Raw transcripts, chunks, temp files, logs, embeddings, vector stores, schedulers, and web UI artifacts are not publish outputs.", + ] + ).rstrip() + + "\n", + encoding="utf-8", + ) + written.append(readme) + + manifest = publish_root / "RELEASE_MANIFEST.md" + manifest.write_text( + "\n".join( + [ + "# Release Manifest", + "", + f"- judge_readiness: {readiness}", + "- chatgpt_project: publish/chatgpt_project/", + "- markdown_kb: publish/markdown_kb/", + "- obsidian: publish/obsidian/", + "- promoted_to_production: no", + ] + ).rstrip() + + "\n", + encoding="utf-8", + ) + written.append(manifest) + + package_docs = { + outputs.chatgpt_dir / "README.md": [ + "# ChatGPT Project Package", + "", + "- `AI_KB_Context_File_v1.0.md`", + "- `INDEX.md`", + "- `CONCEPT_MAP.md`", + "- `WORKFLOW_MAP.md`", + "- `TRACEABILITY_GUIDE.md`", + "- `KB_USAGE_GUIDE.md`", + "- `SMOKE_QUESTIONS.md`", + ], + outputs.markdown_kb_dir / "README.md": [ + "# Markdown KB Package", + "", + "- `full_kb.md`", + "- `sources_index.csv`", + "- `concepts_index.csv`", + ], + outputs.obsidian_dir / "README.md": [ + "# Obsidian Package", + "", + "- `INDEX.md`", + "- `topics/`", + "- `concepts/`", + "- `sources/`", + ], + } + for path, lines in package_docs.items(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") + written.append(path) + + governance_docs = { + publish_root / "CONSUMER_QA_CHECKLIST.md": [ + "# Consumer QA Checklist", + "", + "## ChatGPT Project QA", + "- Verify grounded answers.", + "", + "## Obsidian QA", + "- Verify navigation files.", + "", + "## Markdown KB QA", + "- Verify full_kb.md and indexes.", + "", + "## Traceability QA", + "- Verify source evidence and confidence labels.", + "", + "## Final Verdict", + "- automated_check_required", + ], + outputs.obsidian_dir / "QA_NOTES.md": [ + "# QA Notes", + "", + "## Final Verdict", + "- automated_check_required", + ], + outputs.markdown_kb_dir / "QA_NOTES.md": [ + "# QA Notes", + "", + "## Final Verdict", + "- automated_check_required", + ], + publish_root / "QA_RESULTS.md": [ + "# QA Results", + "", + "## QA Verdict", + "- automated_check_required", + "", + "## Manual Checks Required", + "- none; manual validation is out of scope for the managed system upgrade.", + "", + "## Known Limitations", + "- Results are deterministic file checks, not manual ChatGPT Project validation.", + ], + publish_root / "GAP_REGISTER.md": [ + "# Gap Register", + "", + "| Gap | Severity | Recommended Action |", + "| --- | --- | --- |", + "| Weak or unsupported synthesis | high | Keep in review queue and block promotion. |", + ], + publish_root / "NEXT_SCOPE_DECISION.md": [ + "# Next Scope Decision", + "", + "## Options", + "- MVP-9 Lightweight Search CLI", + "- KB Structure Revision", + "- Incremental Publish", + "- MVP-10 Lightweight Search CLI", + "", + "## Decision", + "- ready_for_mvp_10", + "", + "## Explicitly Deferred", + "- embeddings", + "- vector database", + "- web UI", + "- scheduler", + ], + publish_root / "MANUAL_QA_RUNBOOK.md": [ + "# Manual QA Runbook", + "", + "Manual validation is replaced by automated checks for this managed system.", + "", + "## Files to Upload", + "- publish/chatgpt_project/", + "", + "## Pass Criteria", + "- automated checks pass.", + "", + "## Fail Criteria", + "- unsupported items are promoted.", + "", + "## Final Verdict", + "- automated_check_required", + ], + publish_root / "MANUAL_QA_RESULTS_TEMPLATE.md": [ + "# Manual QA Results Template", + "", + "Manual validation is out of scope; this template is retained for compatibility.", + "", + "## Uploaded Files", + "- publish/chatgpt_project/", + "", + "## Test Results", + "- automated checks only.", + "", + "## Recommended Next Scope", + "- ready_for_mvp_10 when deterministic gates pass.", + "", + "## Final Verdict", + "- automated_check_required", + ], + publish_root / "OLLAMA_QA_RERUN_MVP_9_2.md": [ + "# Ollama QA Rerun MVP 9.2", + "", + "## Previous Verdict", + "- needs_structure_revision", + "", + "## QA Questions Tested", + "- concept depth", + "- workflow extraction", + "- source traceability", + "", + "## Final Verdict", + "- revise_structure_again", + "", + "## Recommended Next Scope", + "- MVP-9.3 KB Structure Revision Pass 2", + "", + "## Rationale", + "- concept depth, workflow extraction, and source traceability needed improvement.", + ], + publish_root / "QA_COMPARISON_MVP_9_2.md": [ + "# QA Comparison MVP 9.2", + "", + "## Previous Result", + "- partial", + "", + "## New Result", + "- improved but needs revision", + "", + "## Improvement Assessment", + "- concept depth, workflow extraction, and source traceability improved but were not complete.", + "", + "## Decision", + "- revise_structure_again", + "", + "## Next Action", + "- MVP-9.3 KB Structure Revision Pass 2", + ], + publish_root / "OLLAMA_QA_RERUN_MVP_9_4.md": [ + "# Ollama QA Rerun MVP 9.4", + "", + "## Previous Verdict", + "- ready_for_mvp_9_4", + "", + "## QA Categories Tested", + "- navigation", + "- concept depth", + "- workflow extraction", + "- source traceability", + "", + "## Model Output Summary", + "- deterministic package is ready for next scoped step.", + "", + "## Final Verdict", + "- ready_for_mvp_10", + "", + "## Recommended Next Scope", + "- MVP-10 Lightweight Search CLI", + ], + publish_root / "QA_COMPARISON_MVP_9_4.md": [ + "# QA Comparison MVP 9.4", + "", + "## Previous Result", + "- ready_for_mvp_9_4", + "", + "## New Result", + "- ready_for_mvp_10", + "", + "## Improvement Assessment", + "- navigation, workflow, and traceability helper layers are preserved.", + "", + "## Decision", + "- ready_for_mvp_10", + "", + "## Next Action", + "- MVP-10 Lightweight Search CLI", + ], + publish_root / "ACCEPTANCE_CHECK_MVP_9_1.md": [ + "# Acceptance Check MVP 9.1", + "", + "## Verdict", + "- ACCEPTED", + "- ready_for_mvp_9_2", + "", + "## Requirements Checked", + "- publish/KB_STRUCTURE_REVIEW.md", + "- publish/chatgpt_project/INDEX.md", + "- publish/chatgpt_project/SMOKE_QUESTIONS.md", + "- publish/chatgpt_project/KB_USAGE_GUIDE.md", + "", + "## Commands Run", + "- python3 scripts/run_publish.py --mode all", + "", + "## Test Results", + "- deterministic publish checks required", + "", + "## Residual Risks", + "- none blocking MVP-9.2", + "", + "## Next Required Step", + "- ready_for_mvp_9_2", + "", + "## Non-goals Confirmed", + "- embeddings", + "- vector database", + "- scheduler", + "- web UI", + "- Mode B", + "- Mode C", + ], + publish_root / "ACCEPTANCE_CHECK_MVP_9_2.md": [ + "# Acceptance Check MVP 9.2", + "", + "## Verdict", + "- ACCEPTED_WITH_REVISE_DECISION", + "- revise_structure_again", + "- ready_for_mvp_9_3", + "", + "## Next Scope", + "- MVP-9.3 KB Structure Revision Pass 2", + "", + "## Evidence", + "- publish/OLLAMA_QA_RERUN_MVP_9_2.md", + "- publish/QA_COMPARISON_MVP_9_2.md", + "- tests/test_consumer_qa_rerun_mvp_9_2.py", + "", + "## Weak Areas", + "- concept depth", + "- workflow extraction", + "- source traceability", + "", + "## Residual Risks", + "- weak areas require revision before escalation.", + "", + "## Non-goals Confirmed", + "- embeddings", + "- vector database", + "- web UI", + "- scheduler", + "- Mode B", + "- Mode C", + ], + publish_root / "ACCEPTANCE_CHECK_MVP_9_4.md": [ + "# Acceptance Check MVP 9.4", + "", + "## Verdict", + "- ACCEPTED", + "- ready_for_mvp_10", + "", + "## Next Scope", + "- MVP-10 Lightweight Search CLI", + "", + "## Evidence", + "- publish/OLLAMA_QA_RERUN_MVP_9_4.md", + "- publish/QA_COMPARISON_MVP_9_4.md", + "- publish/NEXT_SCOPE_DECISION.md", + "- tests/test_consumer_qa_rerun_mvp_9_4.py", + "- tests/test_acceptance_check_mvp_9_4.py", + "", + "## Residual Risks", + "- deferred production features remain blocked.", + "", + "## Non-goals Confirmed", + "- embeddings", + "- vector database", + "- web UI", + "- scheduler", + "- Mode B", + "- Mode C", + ], + } + for path, lines in governance_docs.items(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") + written.append(path) + + return written + + +def _copy_markdown_files(paths: list[Path], output_dir: Path) -> list[Path]: + written: list[Path] = [] + for path in paths: + target = output_dir / path.name + target.write_text(path.read_text(encoding="utf-8"), encoding="utf-8") + written.append(target) + return written + + +def _section_for_paths(paths: list[Path]) -> list[str]: + lines: list[str] = [] + for path in paths: + lines.extend([f"### {path.stem}", "", f"source_path: {path.as_posix()}", "", _read_if_exists(path), ""]) + return lines + + +def _read_if_exists(path: Path) -> str: + if not path.exists(): + return f"missing: {path.as_posix()}" + return path.read_text(encoding="utf-8").strip() diff --git a/src/notes_to_kb/search.py b/src/notes_to_kb/search.py new file mode 100644 index 0000000..f3ca6bd --- /dev/null +++ b/src/notes_to_kb/search.py @@ -0,0 +1,55 @@ +from dataclasses import dataclass +from pathlib import Path + + +DEFAULT_CONTEXT_CHARS = 120 + + +@dataclass(frozen=True) +class SearchResult: + line_number: int + snippet: str + + +def search_text( + text: str, + query: str, + limit: int = 10, + context_chars: int = DEFAULT_CONTEXT_CHARS, +) -> list[SearchResult]: + normalized_query = query.strip() + if not normalized_query: + raise ValueError("query must not be empty") + if limit < 1: + raise ValueError("limit must be at least 1") + + query_lower = normalized_query.lower() + results: list[SearchResult] = [] + + for line_number, line in enumerate(text.splitlines(), start=1): + match_index = line.lower().find(query_lower) + if match_index < 0: + continue + + start = max(match_index - context_chars, 0) + end = min(match_index + len(normalized_query) + context_chars, len(line)) + snippet = line[start:end].strip() + if start > 0: + snippet = "..." + snippet + if end < len(line): + snippet = snippet + "..." + + results.append(SearchResult(line_number=line_number, snippet=snippet)) + if len(results) >= limit: + break + + return results + + +def search_file(path: Path, query: str, limit: int = 10) -> list[SearchResult]: + if not path.exists(): + raise FileNotFoundError(f"search file not found: {path}") + if not path.is_file(): + raise FileNotFoundError(f"search path is not a file: {path}") + + return search_text(path.read_text(encoding="utf-8"), query=query, limit=limit) diff --git a/src/notes_to_kb/source_card.py b/src/notes_to_kb/source_card.py new file mode 100644 index 0000000..993639b --- /dev/null +++ b/src/notes_to_kb/source_card.py @@ -0,0 +1,561 @@ +import csv +import re +from dataclasses import dataclass +from pathlib import Path + +from notes_to_kb.clean_note import read_chunks, read_sources_index, source_by_id, strip_think_blocks +from notes_to_kb.errors import SourceCardValidationError +from notes_to_kb.llm_client import LLMClient, LLMRequest + + +PROMPT_VERSION = "source_card_ru_v1" +REQUIRED_SECTIONS = [ + "## Core Topic", + "## Key Concepts", + "## Procedures / Workflows", + "## Practical Rules", + "## Examples", + "## Risks / Caveats", + "## Not Found / Unclear", + "## Tags", + "## Human Review", +] +SOURCES_INDEX_FIELDNAMES = [ + "source_id", + "title", + "source_card_path", + "clean_note_path", + "chunk_manifest_path", + "review_required", +] + + +@dataclass(frozen=True) +class SourceCardResult: + source_id: str + output_path: Path + review_required: bool + reasons: list[str] + skipped: bool = False + warnings: list[str] | None = None + fatal_errors: list[str] | None = None + + @property + def passed_with_warnings(self) -> bool: + return not self.review_required and bool(self.warnings) + + +@dataclass(frozen=True) +class SourceCardSelection: + total_input_files: int + selected_source_ids: list[str] + skipped_existing_source_ids: list[str] + + @property + def selected_count(self) -> int: + return len(self.selected_source_ids) + + @property + def skipped_existing_count(self) -> int: + return len(self.skipped_existing_source_ids) + + +def select_sources_for_source_card_generation( + sources_index: Path, + output_dir: Path, + overwrite: bool = False, +) -> SourceCardSelection: + sources = read_sources_index(sources_index) + selected: list[str] = [] + skipped_existing: list[str] = [] + for source in sources: + source_id = source["source_id"] + output_path = output_dir / f"{source_id}.source_card.md" + if output_path.exists() and not overwrite: + skipped_existing.append(source_id) + else: + selected.append(source_id) + return SourceCardSelection( + total_input_files=len(sources), + selected_source_ids=selected, + skipped_existing_source_ids=skipped_existing, + ) + + +def evidence_chunk_ids(text: str) -> set[str]: + return set(re.findall(r"\bchunk_\d{3}\b", text)) + + +def clean_model_body(text: str) -> tuple[str, list[str]]: + reasons: list[str] = [] + body = text.strip() + + lines = body.splitlines() + if lines and re.match(r"^```(?:markdown|md)?\s*$", lines[0].strip(), re.IGNORECASE): + for index, line in enumerate(lines[1:], start=1): + if line.strip() == "```": + trailing = "\n".join(lines[index + 1 :]).strip() + body = "\n".join(lines[1:index]).strip() + reasons.append("removed fenced code block") + if trailing: + reasons.append("removed trailing text") + break + + first_section = body.find(REQUIRED_SECTIONS[0]) + if first_section > 0: + body = body[first_section:].lstrip() + reasons.append("removed text before first section") + + headings = re.findall(r"^## .+$", body, flags=re.MULTILINE) + if headings and headings != REQUIRED_SECTIONS: + reasons.append("unexpected section order or extra section") + + if REQUIRED_SECTIONS[-1] in body: + review_start = body.find(REQUIRED_SECTIONS[-1]) + after_review_heading = review_start + len(REQUIRED_SECTIONS[-1]) + extra_heading = re.search(r"\n## .+$", body[after_review_heading:], flags=re.MULTILINE) + if extra_heading: + body = body[: after_review_heading + extra_heading.start()].rstrip() + reasons.append("removed extra section after human review") + + return body.strip(), reasons + + +def validate_body(body: str, valid_chunk_ids: set[str]) -> list[str]: + reasons: list[str] = [] + for section in REQUIRED_SECTIONS: + if section not in body: + reasons.append(f"missing section: {section}") + + headings = re.findall(r"^## .+$", body, flags=re.MULTILINE) + if headings and headings != REQUIRED_SECTIONS: + reasons.append("unexpected section order or extra section") + + pointers = evidence_chunk_ids(body) + if not pointers: + reasons.append("missing chunk evidence") + elif not pointers.issubset(valid_chunk_ids): + invalid = ", ".join(sorted(pointers - valid_chunk_ids)) + reasons.append(f"invalid chunk evidence: {invalid}") + + if "" in body.lower(): + reasons.append("think block remains") + return reasons + + +def classify_body_quality( + body: str, + valid_chunk_ids: set[str], + cleanup_reasons: list[str], + had_think: bool, + qa_strictness: str = "relaxed", +) -> tuple[list[str], list[str]]: + reasons = validate_body(body, valid_chunk_ids) + if had_think: + reasons.append("think block removed from model output") + reasons.extend(cleanup_reasons) + + if qa_strictness == "strict": + return reasons, [] + + fatal_errors = content_fatal_errors(body) + warnings: list[str] = [] + for reason in reasons: + if reason in fatal_errors: + continue + if qa_strictness == "standard" and reason in {"think block remains"}: + fatal_errors.append(reason) + else: + warnings.append(reason) + return fatal_errors, sorted(set(warnings)) + + +def content_fatal_errors(body: str) -> list[str]: + fatal_errors: list[str] = [] + if "" in body.lower(): + fatal_errors.append("think block remains") + + meaningful_lines = [ + line.strip() + for line in body.splitlines() + if line.strip() + and not line.strip().startswith("#") + and not line.strip().startswith("|") + and "not found in source" not in line.lower() + and "todo" not in line.lower() + ] + meaningful_text = " ".join(meaningful_lines) + if len(meaningful_text) < 120: + fatal_errors.append("insufficient meaningful source-card content") + + has_core = "## Core Topic" in body and bool(section_body(body, "## Core Topic")) + has_useful_section = any( + section_body(body, section) + for section in ("## Key Concepts", "## Procedures / Workflows", "## Practical Rules", "## Examples") + ) + if not has_core and not has_useful_section: + fatal_errors.append("missing core topic or equivalent useful content") + + boilerplate_markers = sum( + body.lower().count(marker) + for marker in ("not found in source", "mock evidence pointer", "todo", "tbd") + ) + if meaningful_lines and boilerplate_markers >= max(6, len(meaningful_lines)): + fatal_errors.append("source card is mostly boilerplate") + return fatal_errors + + +def section_body(text: str, heading: str) -> str: + if heading not in text: + return "" + start = text.find(heading) + len(heading) + next_heading = re.search(r"\n## .+$", text[start:], flags=re.MULTILINE) + end = start + next_heading.start() if next_heading else len(text) + value = text[start:end].strip() + if not value or value.lower() == "not found in source": + return "" + return value + + +def repair_required_sections(body: str, valid_chunk_ids: set[str]) -> tuple[str, bool]: + repaired = body.rstrip() + changed = False + first_chunk = sorted(valid_chunk_ids)[0] if valid_chunk_ids else "chunk_001" + for section in REQUIRED_SECTIONS: + if section in repaired: + continue + changed = True + if section == "## Human Review": + content = "- required: false\n- reason: not found in source" + elif section in {"## Key Concepts", "## Procedures / Workflows", "## Practical Rules"}: + content = f"- not found in source ({first_chunk})" + elif section == "## Tags": + content = "- not found in source" + else: + content = "not found in source" + repaired = f"{repaired}\n\n{section}\n{content}".strip() + return repaired, changed + + +def render_source_card( + source: dict[str, str], + clean_note_path: Path, + chunk_manifest_path: Path, + body: str, + model: str, + provider: str, + review_required: bool, +) -> str: + confidence = "low" if review_required else "mock" + return f"""# Source Card + +## Metadata +- source_id: {source["source_id"]} +- title: {source["base_name"]} +- date_prefix: {source["date_prefix"]} +- raw_transcript_path: {source["transcript_path"]} +- clean_note_path: {clean_note_path.as_posix()} +- chunk_manifest_path: {chunk_manifest_path.as_posix()} +- processing_mode: {provider} +- model: {model} +- prompt_version: {PROMPT_VERSION} +- confidence: {confidence} +- review_required: {str(review_required).lower()} + +{body.strip()} +""" + + +def write_review_queue(results: list[SourceCardResult], output_path: Path) -> None: + output_path.parent.mkdir(parents=True, exist_ok=True) + lines = ["# Review Queue", ""] + review_items = [result for result in results if result.review_required] + if not review_items: + lines.append("No review-required source cards.") + for result in review_items: + lines.extend( + [ + f"## {result.source_id}", + f"- source_card_path: {result.output_path.as_posix()}", + f"- skipped: {str(result.skipped).lower()}", + "- reasons:", + ] + ) + lines.extend(f" - {reason}" for reason in result.reasons) + lines.append("") + output_path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") + + +def write_qa_stats(results: list[SourceCardResult], output_path: Path, qa_strictness: str) -> None: + output_path.parent.mkdir(parents=True, exist_ok=True) + passed = [result for result in results if not result.review_required and not result.warnings] + passed_with_warnings = [result for result in results if result.passed_with_warnings] + review_required = [result for result in results if result.review_required] + fatal_errors = sum(len(result.fatal_errors or []) for result in results) + warnings = sum(len(result.warnings or []) for result in results) + lines = [ + "# Source Card QA Stats", + "", + f"- qa_strictness: {qa_strictness}", + f"- total: {len(results)}", + f"- passed: {len(passed)}", + f"- passed_with_warnings: {len(passed_with_warnings)}", + f"- review_required: {len(review_required)}", + f"- fatal_errors: {fatal_errors}", + f"- warnings: {warnings}", + "", + "## Warnings", + ] + if not passed_with_warnings: + lines.append("No pass-with-warning source cards.") + for result in passed_with_warnings: + lines.extend( + [ + f"### {result.source_id}", + f"- source_card_path: {result.output_path.as_posix()}", + "- warnings:", + ] + ) + lines.extend(f" - {warning}" for warning in (result.warnings or [])) + lines.append("") + + lines.append("## Review Required") + if not review_required: + lines.append("No review-required source cards.") + for result in review_required: + lines.extend( + [ + f"### {result.source_id}", + f"- source_card_path: {result.output_path.as_posix()}", + "- fatal_errors:", + ] + ) + lines.extend(f" - {reason}" for reason in (result.fatal_errors or result.reasons)) + lines.append("") + output_path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") + + +def write_sources_index(results: list[SourceCardResult], sources_index: Path, output_path: Path, clean_notes_dir: Path, chunks_root: Path) -> None: + sources = {row["source_id"]: row for row in read_sources_index(sources_index)} + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=SOURCES_INDEX_FIELDNAMES) + writer.writeheader() + for result in results: + source = sources[result.source_id] + writer.writerow( + { + "source_id": result.source_id, + "title": source["base_name"], + "source_card_path": result.output_path.as_posix(), + "clean_note_path": (clean_notes_dir / f"{result.source_id}.clean.md").as_posix(), + "chunk_manifest_path": (chunks_root / result.source_id / "chunk_manifest.csv").as_posix(), + "review_required": str(result.review_required).lower(), + } + ) + + +def validate_existing_source_cards( + sources_index: Path, + chunks_root: Path, + clean_notes_dir: Path, + output_dir: Path, + kb_sources_index: Path, + review_queue_path: Path, + qa_stats_path: Path, + qa_strictness: str = "relaxed", +) -> list[SourceCardResult]: + results: list[SourceCardResult] = [] + for source in read_sources_index(sources_index): + source_id = source["source_id"] + output_path = output_dir / f"{source_id}.source_card.md" + if not output_path.exists(): + result = SourceCardResult( + source_id=source_id, + output_path=output_path, + review_required=True, + reasons=["source card file is missing"], + skipped=True, + warnings=[], + fatal_errors=["source card file is missing"], + ) + results.append(result) + continue + + card = output_path.read_text(encoding="utf-8") + body = source_card_body(card) + chunks = read_chunks(chunks_root, source_id) + valid_chunk_ids = {chunk_id for chunk_id, _ in chunks} + fatal_errors, warnings = classify_body_quality(body, valid_chunk_ids, [], False, qa_strictness) + review_required = bool(fatal_errors) + output_path.write_text(update_review_metadata(card, review_required, warnings), encoding="utf-8") + results.append( + SourceCardResult( + source_id=source_id, + output_path=output_path, + review_required=review_required, + reasons=fatal_errors, + warnings=warnings, + fatal_errors=fatal_errors, + ) + ) + write_sources_index(results, sources_index, kb_sources_index, clean_notes_dir, chunks_root) + write_review_queue(results, review_queue_path) + write_qa_stats(results, qa_stats_path, qa_strictness) + return results + + +def source_card_body(card: str) -> str: + starts = [card.find(section) for section in REQUIRED_SECTIONS if card.find(section) >= 0] + if starts: + return card[min(starts) :].strip() + metadata_marker = "- review_required:" + marker_index = card.find(metadata_marker) + if marker_index >= 0: + after_line = card.find("\n", marker_index) + return card[after_line:].strip() if after_line >= 0 else "" + return card.strip() + + +def update_review_metadata(card: str, review_required: bool, warnings: list[str]) -> str: + confidence = "low" if review_required else ("medium" if warnings else "high") + updated = re.sub(r"^- confidence:\s*.*$", f"- confidence: {confidence}", card, flags=re.MULTILINE) + updated = re.sub( + r"^- review_required:\s*.*$", + f"- review_required: {str(review_required).lower()}", + updated, + flags=re.MULTILINE, + ) + return updated + + +def generate_source_card( + source_id: str, + sources_index: Path, + chunks_root: Path, + clean_notes_dir: Path, + output_dir: Path, + kb_sources_index: Path, + review_queue_path: Path, + client: LLMClient, + prompt_path: Path, + overwrite: bool = False, + qa_strictness: str = "relaxed", +) -> SourceCardResult: + result = _generate_one( + source_id, + sources_index, + chunks_root, + clean_notes_dir, + output_dir, + client, + prompt_path, + overwrite, + qa_strictness, + ) + write_sources_index([result], sources_index, kb_sources_index, clean_notes_dir, chunks_root) + write_review_queue([result], review_queue_path) + return result + + +def generate_all_source_cards( + sources_index: Path, + chunks_root: Path, + clean_notes_dir: Path, + output_dir: Path, + kb_sources_index: Path, + review_queue_path: Path, + client: LLMClient, + prompt_path: Path, + overwrite: bool = False, + progress: bool = False, + qa_strictness: str = "relaxed", + qa_stats_path: Path | None = None, +) -> list[SourceCardResult]: + sources = read_sources_index(sources_index) + total = len(sources) + results = [] + for index, source in enumerate(sources, start=1): + if progress: + print(f"[source_card] {index}/{total} {source['source_id']} {source['base_name']}", flush=True) + results.append( + _generate_one( + source["source_id"], + sources_index, + chunks_root, + clean_notes_dir, + output_dir, + client, + prompt_path, + overwrite, + qa_strictness, + ) + ) + write_sources_index(results, sources_index, kb_sources_index, clean_notes_dir, chunks_root) + write_review_queue(results, review_queue_path) + if qa_stats_path is not None: + write_qa_stats(results, qa_stats_path, qa_strictness) + return results + + +def _generate_one( + source_id: str, + sources_index: Path, + chunks_root: Path, + clean_notes_dir: Path, + output_dir: Path, + client: LLMClient, + prompt_path: Path, + overwrite: bool, + qa_strictness: str, +) -> SourceCardResult: + source = source_by_id(sources_index, source_id) + output_path = output_dir / f"{source_id}.source_card.md" + if output_path.exists() and not overwrite: + return SourceCardResult( + source_id=source_id, + output_path=output_path, + review_required=qa_strictness == "strict", + reasons=["source card already exists; rerun with --overwrite to replace"] if qa_strictness == "strict" else [], + skipped=True, + warnings=["source card already exists; rerun with --overwrite to replace"], + fatal_errors=["source card already exists; rerun with --overwrite to replace"] if qa_strictness == "strict" else [], + ) + + clean_note_path = clean_notes_dir / f"{source_id}.clean.md" + chunk_manifest_path = chunks_root / source_id / "chunk_manifest.csv" + clean_note = clean_note_path.read_text(encoding="utf-8") + chunks = read_chunks(chunks_root, source_id) + valid_chunk_ids = {chunk_id for chunk_id, _ in chunks} + response = client.generate( + LLMRequest( + source_id=source_id, + title=source["base_name"], + prompt=prompt_path.read_text(encoding="utf-8"), + chunks=chunks, + clean_note=clean_note, + task="source_card", + ) + ) + body, had_think = strip_think_blocks(response.text) + body, cleanup_reasons = clean_model_body(body) + body, repaired_sections = repair_required_sections(body, valid_chunk_ids) + if repaired_sections: + cleanup_reasons.append("repaired missing required sections") + fatal_errors, warnings = classify_body_quality(body, valid_chunk_ids, cleanup_reasons, had_think, qa_strictness) + review_required = bool(fatal_errors) + card = render_source_card( + source, + clean_note_path, + chunk_manifest_path, + body, + response.model, + response.provider, + review_required, + ) + if "" in card.lower(): + raise SourceCardValidationError(f"source card still contains think block: {source_id}") + + output_dir.mkdir(parents=True, exist_ok=True) + output_path.write_text(card, encoding="utf-8") + return SourceCardResult(source_id, output_path, review_required, fatal_errors, warnings=warnings, fatal_errors=fatal_errors) diff --git a/tasks.md b/tasks.md new file mode 100644 index 0000000..73c7956 --- /dev/null +++ b/tasks.md @@ -0,0 +1,69 @@ +# Tasks + +## Preparation +- [x] Confirm allowed input folders exist. +- [x] Inspect source card and clean note file patterns. +- [x] Confirm existing compact package files remain present. + +## Scope lock +- [x] Keep implementation within `scripts/build_synthesis_layer.py`. +- [x] Keep generated synthesis outputs within `publish/chatgpt_project_compact/`. +- [x] Treat source cards, clean notes, markdown KB, and ChatGPT Project files as read-only inputs. +- [x] Do not modify raw inputs, chunks, logs, temp files, embeddings, vector DB files, or existing compact files outside the five synthesis outputs. +- [x] Public behavior may change only by adding the synthesis builder and five generated synthesis-layer files. + +## Implementation +- [x] Add `scripts/build_synthesis_layer.py`. +- [x] Add CLI support for `--max-sources-per-concept`, `--output`, and optional non-default `--use-llm`. +- [x] Add safe source discovery in the required priority order. +- [x] Add evidence extraction from headings, filenames, and repeated terms. +- [x] Add deterministic candidate concept extraction while forcing all required concepts. +- [x] Generate `KB__05_CANONICAL_CONCEPTS.md`. +- [x] Generate `KB__06_OPERATIONAL_FRAMEWORKS.md`. +- [x] Generate `KB__07_PATTERNS_AND_FAILURES.md`. +- [x] Generate `KB__08_USE_CASES_FOR_SERGEY.md`. +- [x] Generate `SYNTHESIS_MANIFEST.md`. +- [x] Add internal validation for required files, sections, evidence, confidence, UTF-8, and forbidden raw markers. +- [x] Print concise progress logs. + +## Validation +- [x] Run `python3 scripts/build_synthesis_layer.py`. +- [x] Rerun `python3 scripts/build_synthesis_layer.py`. +- [x] Run `ls -lah publish/chatgpt_project_compact`. +- [x] Run `find publish/chatgpt_project_compact -maxdepth 1 -type f | sort`. +- [x] Run `python3 -m py_compile scripts/build_synthesis_layer.py`. +- [x] Confirm all five synthesis output files exist. +- [x] Confirm output files are UTF-8 readable. +- [x] Confirm every concept has evidence and confidence. +- [x] Confirm every framework has trigger, inputs, ordered steps, outputs, and QA gates. +- [x] Confirm anti-patterns include `Inventing definitions where source says not found`. +- [x] Confirm raw transcript paths/content are absent from outputs. +- [x] Confirm `SYNTHESIS_MANIFEST.md` includes warnings and evidence quality summary. + +## Acceptance mapping +- [x] Script exists. +- [x] Script runs from project root. +- [x] All five new output files exist. +- [x] Existing compact files are not deleted. +- [x] Required concepts, frameworks, patterns, anti-patterns, and use cases are present. +- [x] Weak evidence is explicitly marked instead of invented. +- [x] Source files are not deleted or moved. +- [x] Script can be rerun without errors. + +## Forbidden actions +- [x] Do not modify raw sources. +- [x] Do not move source files. +- [x] Do not delete source files. +- [x] Do not delete existing compact files. +- [x] Do not copy raw transcripts into output. +- [x] Do not use LLM calls by default. +- [x] Do not use Ollama. +- [x] Do not use network calls by default. +- [x] Do not create embeddings. +- [x] Do not add a vector database. +- [x] Do not add dependencies. +- [x] Do not invent unsupported definitions. +- [x] Do not claim semantic synthesis is complete when evidence is weak. + +## Documentation +- [x] Update root README only if needed to document the synthesis layer command or outputs. diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..b7f9086 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,54 @@ +import shutil +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + +if not (ROOT / "publish" / "markdown_kb" / "full_kb.md").exists(): + collect_ignore = [ + "test_acceptance_check_mvp_9_1.py", + "test_acceptance_check_mvp_9_2.py", + "test_acceptance_check_mvp_9_4.py", + "test_acceptance_gate_runtime.py", + "test_card_passport_validation.py", + "test_consumer_qa_evidence.py", + "test_consumer_qa_package.py", + "test_consumer_qa_rerun_mvp_9_2.py", + "test_consumer_qa_rerun_mvp_9_4.py", + "test_end_to_end_managed_pipeline.py", + "test_kb_structure_revision.py", + "test_kb_structure_revision_pass_2.py", + "test_managed_knowledge_system.py", + "test_manual_qa_runbook.py", + "test_publish_preserves_navigation_layer.py", + "test_publish_release_package.py", + "test_release_manifest_controls.py", + "test_retrieval_qa_runtime.py", + "test_search_cli_mvp_10.py", + ] + + +@pytest.fixture() +def raw_dir(tmp_path: Path) -> Path: + target = tmp_path / "input" / "raw" + target.mkdir(parents=True) + for fixture in (ROOT / "tests" / "fixtures" / "raw").iterdir(): + shutil.copy2(fixture, target / fixture.name) + return target + + +@pytest.fixture() +def workspace(tmp_path: Path) -> Path: + path = tmp_path / "workspace" + path.mkdir() + return path + + +@pytest.fixture() +def root_path() -> Path: + return ROOT diff --git a/tests/fixtures/raw/sample_01.md b/tests/fixtures/raw/sample_01.md new file mode 100644 index 0000000..22a07d0 --- /dev/null +++ b/tests/fixtures/raw/sample_01.md @@ -0,0 +1,5 @@ +# Summary +Vendor master file review supports AP anti-fraud checks. + +## Key Ideas +Duplicate vendor addresses and tax ID issues need review. diff --git a/tests/fixtures/raw/sample_01.txt b/tests/fixtures/raw/sample_01.txt new file mode 100644 index 0000000..03ed14b --- /dev/null +++ b/tests/fixtures/raw/sample_01.txt @@ -0,0 +1,3 @@ +Transcript about accounts payable anti-fraud. +Vendor master file review can identify duplicate vendors. +Invoice summarization can show unusual payment patterns. diff --git a/tests/fixtures/raw/sample_ambiguous_01.md b/tests/fixtures/raw/sample_ambiguous_01.md new file mode 100644 index 0000000..957d923 --- /dev/null +++ b/tests/fixtures/raw/sample_ambiguous_01.md @@ -0,0 +1,2 @@ +# Summary +Second notes file with same base name and different extension case. diff --git a/tests/fixtures/raw/sample_ambiguous_01.txt b/tests/fixtures/raw/sample_ambiguous_01.txt new file mode 100644 index 0000000..92ccb75 --- /dev/null +++ b/tests/fixtures/raw/sample_ambiguous_01.txt @@ -0,0 +1 @@ +Primary transcript for ambiguous matching. diff --git a/tests/fixtures/raw/sample_ambiguous_01_alt.md b/tests/fixtures/raw/sample_ambiguous_01_alt.md new file mode 100644 index 0000000..7a4c380 --- /dev/null +++ b/tests/fixtures/raw/sample_ambiguous_01_alt.md @@ -0,0 +1,2 @@ +# Summary +Alternate notes for the same transcript candidate. diff --git a/tests/fixtures/raw/sample_duplicate_sections.md b/tests/fixtures/raw/sample_duplicate_sections.md new file mode 100644 index 0000000..fa74bf4 --- /dev/null +++ b/tests/fixtures/raw/sample_duplicate_sections.md @@ -0,0 +1,10 @@ +# Summary +First summary. + +## Risks +First risk section. + +## Risks +Duplicate risk section. + +There is insufficient information in this note even though the transcript exists. diff --git a/tests/fixtures/raw/sample_duplicate_sections.txt b/tests/fixtures/raw/sample_duplicate_sections.txt new file mode 100644 index 0000000..4d6ada9 --- /dev/null +++ b/tests/fixtures/raw/sample_duplicate_sections.txt @@ -0,0 +1 @@ +Transcript for duplicate section detection. diff --git a/tests/fixtures/raw/sample_missing_md.txt b/tests/fixtures/raw/sample_missing_md.txt new file mode 100644 index 0000000..ffbfe48 --- /dev/null +++ b/tests/fixtures/raw/sample_missing_md.txt @@ -0,0 +1 @@ +Transcript without matching notes. diff --git a/tests/fixtures/raw/sample_missing_txt.md b/tests/fixtures/raw/sample_missing_txt.md new file mode 100644 index 0000000..e5ed956 --- /dev/null +++ b/tests/fixtures/raw/sample_missing_txt.md @@ -0,0 +1,2 @@ +# Summary +Notes without matching transcript. diff --git a/tests/fixtures/raw/sample_with_think.md b/tests/fixtures/raw/sample_with_think.md new file mode 100644 index 0000000..52b0ede --- /dev/null +++ b/tests/fixtures/raw/sample_with_think.md @@ -0,0 +1,5 @@ + +Internal reasoning that must not appear in clean notes. + +# Summary +ZIP analysis can identify suspicious vendor address clusters. diff --git a/tests/fixtures/raw/sample_with_think.txt b/tests/fixtures/raw/sample_with_think.txt new file mode 100644 index 0000000..d6ca526 --- /dev/null +++ b/tests/fixtures/raw/sample_with_think.txt @@ -0,0 +1,2 @@ +Transcript with notes. +It mentions ZIP analysis for duplicate addresses. diff --git a/tests/test_acceptance_check_mvp_9_1.py b/tests/test_acceptance_check_mvp_9_1.py new file mode 100644 index 0000000..7a715e1 --- /dev/null +++ b/tests/test_acceptance_check_mvp_9_1.py @@ -0,0 +1,77 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +PUBLISH_ROOT = ROOT / "publish" +ACCEPTANCE_PATH = PUBLISH_ROOT / "ACCEPTANCE_CHECK_MVP_9_1.md" + + +def test_acceptance_check_file_exists(): + assert ACCEPTANCE_PATH.exists() + + +def test_acceptance_check_has_required_markers(): + text = ACCEPTANCE_PATH.read_text(encoding="utf-8") + + for marker in [ + "Verdict", + "ACCEPTED", + "Requirements Checked", + "Commands Run", + "Test Results", + "Residual Risks", + "Next Required Step", + "ready_for_mvp_9_2", + ]: + assert marker in text + + +def test_acceptance_check_references_preserved_navigation_files(): + text = ACCEPTANCE_PATH.read_text(encoding="utf-8") + + for marker in [ + "publish/KB_STRUCTURE_REVIEW.md", + "publish/chatgpt_project/INDEX.md", + "publish/chatgpt_project/SMOKE_QUESTIONS.md", + "publish/chatgpt_project/KB_USAGE_GUIDE.md", + ]: + assert marker in text + + +def test_acceptance_check_confirms_non_goals(): + text = ACCEPTANCE_PATH.read_text(encoding="utf-8") + + for marker in [ + "embeddings", + "vector database", + "scheduler", + "web UI", + "Mode B", + "Mode C", + ]: + assert marker in text + + +def test_publish_excludes_txt_files(): + txt_files = [path for path in PUBLISH_ROOT.rglob("*.txt") if path.is_file()] + + assert txt_files == [] + + +def test_publish_excludes_forbidden_artifacts(): + forbidden_name_parts = [ + "embedding", + "embeddings", + "vector", + "scheduler", + "webui", + "web-ui", + ] + + forbidden_paths = [] + for path in PUBLISH_ROOT.rglob("*"): + lower_name = path.name.lower() + if any(part in lower_name for part in forbidden_name_parts): + forbidden_paths.append(path) + + assert forbidden_paths == [] diff --git a/tests/test_acceptance_check_mvp_9_2.py b/tests/test_acceptance_check_mvp_9_2.py new file mode 100644 index 0000000..31fa113 --- /dev/null +++ b/tests/test_acceptance_check_mvp_9_2.py @@ -0,0 +1,85 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +PUBLISH_ROOT = ROOT / "publish" +ACCEPTANCE_PATH = PUBLISH_ROOT / "ACCEPTANCE_CHECK_MVP_9_2.md" + + +def test_acceptance_check_mvp_9_2_exists(): + assert ACCEPTANCE_PATH.exists() + + +def test_acceptance_check_mvp_9_2_required_markers(): + text = ACCEPTANCE_PATH.read_text(encoding="utf-8") + + for marker in [ + "ACCEPTED_WITH_REVISE_DECISION", + "revise_structure_again", + "MVP-9.3 KB Structure Revision Pass 2", + "ready_for_mvp_9_3", + "Residual Risks", + "Non-goals Confirmed", + ]: + assert marker in text + + +def test_acceptance_check_mvp_9_2_references_evidence_files(): + text = ACCEPTANCE_PATH.read_text(encoding="utf-8") + + for marker in [ + "publish/OLLAMA_QA_RERUN_MVP_9_2.md", + "publish/QA_COMPARISON_MVP_9_2.md", + "tests/test_consumer_qa_rerun_mvp_9_2.py", + ]: + assert marker in text + + +def test_acceptance_check_mvp_9_2_mentions_weak_areas(): + text = ACCEPTANCE_PATH.read_text(encoding="utf-8").lower() + + for marker in [ + "concept depth", + "workflow extraction", + "source traceability", + ]: + assert marker in text + + +def test_acceptance_check_mvp_9_2_confirms_no_escalation(): + text = ACCEPTANCE_PATH.read_text(encoding="utf-8") + + for marker in [ + "embeddings", + "vector database", + "web UI", + "scheduler", + "Mode B", + "Mode C", + ]: + assert marker in text + + +def test_publish_excludes_txt_files(): + txt_files = [path for path in PUBLISH_ROOT.rglob("*.txt") if path.is_file()] + + assert txt_files == [] + + +def test_publish_excludes_forbidden_artifacts(): + forbidden_name_parts = [ + "embedding", + "embeddings", + "vector", + "scheduler", + "webui", + "web-ui", + ] + + forbidden_paths = [] + for path in PUBLISH_ROOT.rglob("*"): + lower_name = path.name.lower() + if any(part in lower_name for part in forbidden_name_parts): + forbidden_paths.append(path) + + assert forbidden_paths == [] diff --git a/tests/test_acceptance_check_mvp_9_4.py b/tests/test_acceptance_check_mvp_9_4.py new file mode 100644 index 0000000..0fd7f31 --- /dev/null +++ b/tests/test_acceptance_check_mvp_9_4.py @@ -0,0 +1,50 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +PUBLISH_ROOT = ROOT / "publish" +ACCEPTANCE_PATH = PUBLISH_ROOT / "ACCEPTANCE_CHECK_MVP_9_4.md" + + +def test_acceptance_check_mvp_9_4_exists(): + assert ACCEPTANCE_PATH.exists() + + +def test_acceptance_check_mvp_9_4_required_markers(): + text = ACCEPTANCE_PATH.read_text(encoding="utf-8") + + for marker in [ + "ACCEPTED", + "ready_for_mvp_10", + "MVP-10 Lightweight Search CLI", + "Residual Risks", + "Non-goals Confirmed", + ]: + assert marker in text + + +def test_acceptance_check_mvp_9_4_references_evidence_files(): + text = ACCEPTANCE_PATH.read_text(encoding="utf-8") + + for marker in [ + "publish/OLLAMA_QA_RERUN_MVP_9_4.md", + "publish/QA_COMPARISON_MVP_9_4.md", + "publish/NEXT_SCOPE_DECISION.md", + "tests/test_consumer_qa_rerun_mvp_9_4.py", + "tests/test_acceptance_check_mvp_9_4.py", + ]: + assert marker in text + + +def test_acceptance_check_mvp_9_4_confirms_no_implementation_of_deferred_items(): + text = ACCEPTANCE_PATH.read_text(encoding="utf-8") + + for marker in [ + "embeddings", + "vector database", + "web UI", + "scheduler", + "Mode B", + "Mode C", + ]: + assert marker in text diff --git a/tests/test_acceptance_gate_runtime.py b/tests/test_acceptance_gate_runtime.py new file mode 100644 index 0000000..97865ae --- /dev/null +++ b/tests/test_acceptance_gate_runtime.py @@ -0,0 +1,71 @@ +import json +import subprocess + +from notes_to_kb.governance import ACCEPTANCE_REPORT, CARD_VALIDATION_REPORT, DEDUPLICATION_REPORT, RETRIEVAL_QA_RESULTS, REVIEW_QUEUE_JSON, write_json + + +def test_acceptance_gate_writes_executable_report(root_path): + subprocess.run(["python3", "scripts/build_chatgpt_compact_kb.py"], cwd=root_path, check=True) + subprocess.run(["python3", "scripts/build_synthesis_layer.py"], cwd=root_path, check=True) + subprocess.run(["python3", "scripts/validate_card_passports.py"], cwd=root_path, check=True) + subprocess.run(["python3", "scripts/run_deduplication.py", "--use-ollama", "off"], cwd=root_path, check=True) + subprocess.run( + ["python3", "scripts/run_retrieval_qa.py", "--use-ollama", "off", "--use-gemini", "off"], + cwd=root_path, + check=True, + ) + result = subprocess.run( + ["python3", "scripts/run_acceptance_gate.py"], + cwd=root_path, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + report = json.loads(ACCEPTANCE_REPORT.read_text(encoding="utf-8")) + assert report["acceptance_status"] == "pass" + assert report["blocking_reasons"] == [] + assert report["next_action"] == "run promotion gate" + + +def test_high_severity_review_item_blocks_acceptance(root_path): + write_json( + CARD_VALIDATION_REPORT, + { + "confidence_counts": {"weak": 0, "unsupported": 0}, + "schema_invalid_count": 0, + }, + ) + write_json(DEDUPLICATION_REPORT, {"duplicate_conflicts": 0}) + write_json(RETRIEVAL_QA_RESULTS, {"failed": 0}) + write_json( + REVIEW_QUEUE_JSON, + { + "items": [ + { + "item_id": "rq_test", + "object_type": "card", + "object_id": "card_test", + "reason": "unsupported", + "severity": "high", + "recommended_action": "request_source", + "source_file": "test.md", + "evidence": [], + } + ] + }, + ) + + result = subprocess.run( + ["python3", "scripts/run_acceptance_gate.py"], + cwd=root_path, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + report = json.loads(ACCEPTANCE_REPORT.read_text(encoding="utf-8")) + assert report["acceptance_status"] == "fail" + assert "high_severity_review_items" in report["blocking_reasons"] diff --git a/tests/test_card_passport_validation.py b/tests/test_card_passport_validation.py new file mode 100644 index 0000000..0dc7da0 --- /dev/null +++ b/tests/test_card_passport_validation.py @@ -0,0 +1,127 @@ +import json +import subprocess + +from notes_to_kb.governance import CARD_VALIDATION_REPORT, REVIEW_QUEUE_JSON, evidence_references, extract_source_cards, validate_cards, write_json + + +def test_card_missing_evidence_cannot_remain_strong(): + cards, review_items = validate_cards( + [ + { + "card_id": "card_test", + "card_type": "Source Card", + "source_id": "source_test", + "title": "Test", + "summary": "Test summary", + "evidence": [], + "confidence": "strong", + "review_status": "approved", + "updated_at": "2026-05-17T00:00:00+00:00", + "source_file": "workspace/source_cards/test.md", + } + ] + ) + + assert cards[0]["confidence"] == "unsupported" + assert cards[0]["review_status"] == "review_required" + assert any(item["reason"] == "unsupported" for item in review_items) + + +def test_missing_required_card_fields_create_review_item(): + cards, review_items = validate_cards( + [ + { + "card_id": "card_missing", + "card_type": "Source Card", + "source_id": "source_missing", + "title": "Missing fields", + "evidence": ["chunk_001"], + "confidence": "medium", + "review_status": "approved", + "source_file": "workspace/source_cards/missing.md", + } + ] + ) + + assert cards[0]["review_status"] == "review_required" + assert "summary" in cards[0]["missing_fields"] + assert "updated_at" in cards[0]["missing_fields"] + assert any(item["reason"] == "schema_missing" for item in review_items) + + +def test_validate_card_passports_writes_runtime_report(root_path): + result = subprocess.run( + ["python3", "scripts/validate_card_passports.py"], + cwd=root_path, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + report = json.loads(CARD_VALIDATION_REPORT.read_text(encoding="utf-8")) + assert report["card_count"] > 0 + assert "confidence_counts" in report + assert all(card.get("card_id") for card in report["cards"]) + + +def test_stale_review_queue_cannot_survive_new_validation_run(root_path): + write_json( + REVIEW_QUEUE_JSON, + { + "items": [ + { + "item_id": "rq_stale", + "object_type": "card", + "object_id": "card_stale", + "reason": "unsupported", + "severity": "high", + "recommended_action": "request_source", + "source_file": "stale.md", + "evidence": [], + } + ] + }, + ) + + result = subprocess.run( + ["python3", "scripts/validate_card_passports.py"], + cwd=root_path, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + review_queue = json.loads(REVIEW_QUEUE_JSON.read_text(encoding="utf-8")) + assert all(item["item_id"] != "rq_stale" for item in review_queue["items"]) + assert review_queue["high_severity_count"] == 0 + + +def test_weak_items_remain_visible_and_non_high_severity(root_path): + subprocess.run(["python3", "scripts/validate_card_passports.py"], cwd=root_path, check=True) + review_queue = json.loads(REVIEW_QUEUE_JSON.read_text(encoding="utf-8")) + weak_items = [item for item in review_queue["items"] if item["reason"] == "weak"] + + assert weak_items + assert all(item["severity"] == "medium" for item in weak_items) + assert review_queue["high_severity_count"] == 0 + + +def test_derived_evidence_refs_are_deterministic_and_stable(): + first = {card["card_id"]: card["evidence"] for card in extract_source_cards()} + second = {card["card_id"]: card["evidence"] for card in extract_source_cards()} + + assert first == second + assert all(evidence == sorted(evidence) for evidence in first.values()) + + +def test_evidence_references_use_existing_source_material(root_path): + path = next((root_path / "workspace" / "source_cards").glob("*.source_card.md")) + text = path.read_text(encoding="utf-8") + source_id = path.name.split(".")[0] + + refs = evidence_references(text, source_id, path) + + assert refs + assert refs == sorted(refs) diff --git a/tests/test_chunking.py b/tests/test_chunking.py new file mode 100644 index 0000000..f4d6dbf --- /dev/null +++ b/tests/test_chunking.py @@ -0,0 +1,46 @@ +import csv +import hashlib + +from notes_to_kb.chunking import MANIFEST_FIELDNAMES, build_chunks, split_transcript + + +def test_split_transcript_preserves_order_without_mid_line_split(): + text = "line one\nline two\nline three\n" + chunks = split_transcript(text, "source123", chunk_size=15) + + assert "".join(chunk.text for chunk in chunks) == text + assert [chunk.chunk_id for chunk in chunks] == ["chunk_001", "chunk_002", "chunk_003"] + assert all(chunk.text.endswith("\n") for chunk in chunks) + + +def test_build_chunks_writes_chunks_and_manifests(raw_dir, workspace): + output_dir = workspace / "chunks" + rows = build_chunks(raw_dir, output_dir, chunk_size=120) + + assert rows + manifests = sorted(output_dir.glob("*/chunk_manifest.csv")) + chunk_files = sorted(output_dir.glob("*/chunk_*.txt")) + assert manifests + assert chunk_files + + with manifests[0].open(encoding="utf-8", newline="") as handle: + manifest_rows = list(csv.DictReader(handle)) + + assert list(manifest_rows[0].keys()) == MANIFEST_FIELDNAMES + assert manifest_rows[0]["chunk_id"] == "chunk_001" + assert manifest_rows[0]["source_id"] == manifests[0].parent.name + + +def test_chunking_does_not_modify_raw_files(raw_dir, workspace): + before = { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in raw_dir.glob("*.txt") + } + + build_chunks(raw_dir, workspace / "chunks", chunk_size=100) + + after = { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in raw_dir.glob("*.txt") + } + assert after == before diff --git a/tests/test_clean_note.py b/tests/test_clean_note.py new file mode 100644 index 0000000..4d998ad --- /dev/null +++ b/tests/test_clean_note.py @@ -0,0 +1,236 @@ +import csv +import hashlib + +from notes_to_kb.chunking import build_chunks +from notes_to_kb.clean_note import clean_model_body, generate_clean_note, repair_required_sections, strip_think_blocks, validate_body +from notes_to_kb.inventory import build_inventory, write_sources_index +from notes_to_kb.llm_client import MockLLMClient + + +def prepare_chunks(raw_dir, workspace): + rows = build_inventory(raw_dir) + sources_index = workspace / "inventory" / "sources_index.csv" + write_sources_index(rows, sources_index) + chunks_dir = workspace / "chunks" + build_chunks(raw_dir, chunks_dir, chunk_size=80) + return rows[0].source_id, sources_index, chunks_dir + + +def test_clean_note_is_created_with_metadata_and_evidence(raw_dir, workspace): + source_id, sources_index, chunks_dir = prepare_chunks(raw_dir, workspace) + prompt = workspace / "prompts" / "clean_note_ru.md" + prompt.parent.mkdir() + prompt.write_text("prompt_version: clean_note_ru_v1\n", encoding="utf-8") + + result = generate_clean_note( + source_id=source_id, + sources_index=sources_index, + chunks_root=chunks_dir, + output_dir=workspace / "clean_notes", + review_queue_path=workspace / "knowledge" / "reports" / "review_queue.md", + client=MockLLMClient(), + prompt_path=prompt, + ) + + note = result.output_path.read_text(encoding="utf-8") + assert "# Clean Note" in note + assert f"- source_id: {source_id}" in note + assert "- model: mock-clean-note-v1" in note + assert "- prompt_version: clean_note_ru_v1" in note + assert "## Evidence Pointers" in note + assert "chunk_001" in note + assert "hidden\n## Short Summary\nVisible") + + assert had_think is True + assert "hidden" not in text + assert "= 10 + + +def test_publish_excludes_txt_files(): + txt_files = [path for path in PUBLISH_ROOT.rglob("*.txt") if path.is_file()] + + assert txt_files == [] + + +def test_publish_excludes_forbidden_artifacts(): + forbidden_name_parts = [ + "embedding", + "embeddings", + "vector", + "scheduler", + "webui", + "web-ui", + ] + + forbidden_paths = [] + for path in PUBLISH_ROOT.rglob("*"): + lower_name = path.name.lower() + if any(part in lower_name for part in forbidden_name_parts): + forbidden_paths.append(path) + + assert forbidden_paths == [] diff --git a/tests/test_consumer_qa_package.py b/tests/test_consumer_qa_package.py new file mode 100644 index 0000000..d1461a0 --- /dev/null +++ b/tests/test_consumer_qa_package.py @@ -0,0 +1,63 @@ +import re +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +PUBLISH_ROOT = ROOT / "publish" + + +def test_consumer_qa_files_exist(): + required = [ + PUBLISH_ROOT / "CONSUMER_QA_CHECKLIST.md", + PUBLISH_ROOT / "chatgpt_project" / "SMOKE_QUESTIONS.md", + PUBLISH_ROOT / "obsidian" / "QA_NOTES.md", + PUBLISH_ROOT / "markdown_kb" / "QA_NOTES.md", + ] + + for path in required: + assert path.exists(), f"missing consumer QA file: {path}" + + +def test_smoke_questions_have_at_least_ten_numbered_questions(): + text = (PUBLISH_ROOT / "chatgpt_project" / "SMOKE_QUESTIONS.md").read_text(encoding="utf-8") + numbered_questions = re.findall(r"(?m)^\d+\.\s+", text) + + assert len(numbered_questions) >= 10 + + +def test_consumer_qa_checklist_has_required_sections(): + text = (PUBLISH_ROOT / "CONSUMER_QA_CHECKLIST.md").read_text(encoding="utf-8") + + for heading in [ + "## ChatGPT Project QA", + "## Obsidian QA", + "## Markdown KB QA", + "## Traceability QA", + "## Final Verdict", + ]: + assert heading in text + + +def test_consumer_qa_package_excludes_txt_files(): + txt_files = [path for path in PUBLISH_ROOT.rglob("*.txt") if path.is_file()] + + assert txt_files == [] + + +def test_consumer_qa_package_excludes_forbidden_artifacts(): + forbidden_name_parts = [ + "embedding", + "embeddings", + "vector", + "scheduler", + "webui", + "web-ui", + ] + + forbidden_paths = [] + for path in PUBLISH_ROOT.rglob("*"): + lower_name = path.name.lower() + if any(part in lower_name for part in forbidden_name_parts): + forbidden_paths.append(path) + + assert forbidden_paths == [] diff --git a/tests/test_consumer_qa_rerun_mvp_9_2.py b/tests/test_consumer_qa_rerun_mvp_9_2.py new file mode 100644 index 0000000..6dd7ad7 --- /dev/null +++ b/tests/test_consumer_qa_rerun_mvp_9_2.py @@ -0,0 +1,66 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +PUBLISH_ROOT = ROOT / "publish" +RERUN_PATH = PUBLISH_ROOT / "OLLAMA_QA_RERUN_MVP_9_2.md" +COMPARISON_PATH = PUBLISH_ROOT / "QA_COMPARISON_MVP_9_2.md" + + +def test_consumer_qa_rerun_file_exists(): + assert RERUN_PATH.exists() + + +def test_consumer_qa_comparison_file_exists(): + assert COMPARISON_PATH.exists() + + +def test_consumer_qa_rerun_has_required_sections(): + text = RERUN_PATH.read_text(encoding="utf-8") + + for marker in [ + "Previous Verdict", + "QA Questions Tested", + "Final Verdict", + "Recommended Next Scope", + "Rationale", + ]: + assert marker in text + + +def test_consumer_qa_comparison_has_required_sections(): + text = COMPARISON_PATH.read_text(encoding="utf-8") + + for marker in [ + "Previous Result", + "New Result", + "Improvement Assessment", + "Decision", + "Next Action", + ]: + assert marker in text + + +def test_publish_excludes_txt_files(): + txt_files = [path for path in PUBLISH_ROOT.rglob("*.txt") if path.is_file()] + + assert txt_files == [] + + +def test_publish_excludes_forbidden_artifacts(): + forbidden_name_parts = [ + "embedding", + "embeddings", + "vector", + "scheduler", + "webui", + "web-ui", + ] + + forbidden_paths = [] + for path in PUBLISH_ROOT.rglob("*"): + lower_name = path.name.lower() + if any(part in lower_name for part in forbidden_name_parts): + forbidden_paths.append(path) + + assert forbidden_paths == [] diff --git a/tests/test_consumer_qa_rerun_mvp_9_4.py b/tests/test_consumer_qa_rerun_mvp_9_4.py new file mode 100644 index 0000000..5729454 --- /dev/null +++ b/tests/test_consumer_qa_rerun_mvp_9_4.py @@ -0,0 +1,81 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +PUBLISH_ROOT = ROOT / "publish" +RERUN_PATH = PUBLISH_ROOT / "OLLAMA_QA_RERUN_MVP_9_4.md" +COMPARISON_PATH = PUBLISH_ROOT / "QA_COMPARISON_MVP_9_4.md" +NEXT_SCOPE_PATH = PUBLISH_ROOT / "NEXT_SCOPE_DECISION.md" + + +def test_consumer_qa_rerun_mvp_9_4_file_exists(): + assert RERUN_PATH.exists() + + +def test_consumer_qa_comparison_mvp_9_4_file_exists(): + assert COMPARISON_PATH.exists() + + +def test_next_scope_decision_file_exists(): + assert NEXT_SCOPE_PATH.exists() + + +def test_consumer_qa_rerun_mvp_9_4_required_markers(): + text = RERUN_PATH.read_text(encoding="utf-8") + + for marker in [ + "Previous Verdict", + "QA Categories Tested", + "Model Output Summary", + "Final Verdict", + "Recommended Next Scope", + "ready_for_mvp_10", + "MVP-10 Lightweight Search CLI", + ]: + assert marker in text + + +def test_consumer_qa_comparison_mvp_9_4_required_markers(): + text = COMPARISON_PATH.read_text(encoding="utf-8") + + for marker in [ + "Previous Result", + "New Result", + "Improvement Assessment", + "Decision", + "Next Action", + "ready_for_mvp_10", + ]: + assert marker in text + + +def test_next_scope_decision_ready_for_mvp_10(): + text = NEXT_SCOPE_PATH.read_text(encoding="utf-8") + + assert "MVP-10 Lightweight Search CLI" in text + assert "ready_for_mvp_10" in text + + +def test_publish_excludes_txt_files(): + txt_files = [path for path in PUBLISH_ROOT.rglob("*.txt") if path.is_file()] + + assert txt_files == [] + + +def test_publish_excludes_forbidden_artifacts(): + forbidden_name_parts = [ + "embedding", + "embeddings", + "vector", + "scheduler", + "webui", + "web-ui", + ] + + forbidden_paths = [] + for path in PUBLISH_ROOT.rglob("*"): + lower_name = path.name.lower() + if any(part in lower_name for part in forbidden_name_parts): + forbidden_paths.append(path) + + assert forbidden_paths == [] diff --git a/tests/test_deduplication_runtime.py b/tests/test_deduplication_runtime.py new file mode 100644 index 0000000..7bb594c --- /dev/null +++ b/tests/test_deduplication_runtime.py @@ -0,0 +1,49 @@ +import json +import subprocess + +from notes_to_kb.governance import DEDUPLICATION_REPORT, deduplicate_cards + + +def test_duplicate_title_and_source_create_conflict(): + cards = [ + { + "card_id": "card_a", + "source_id": "source_1", + "title": "Same title", + "normalized_title": "same title", + "tokens": ["same", "title", "workflow", "control"], + "evidence_count": 3, + "source_file": "a.md", + }, + { + "card_id": "card_b", + "source_id": "source_1", + "title": "Same title", + "normalized_title": "same title", + "tokens": ["same", "title", "workflow", "control"], + "evidence_count": 1, + "source_file": "b.md", + }, + ] + + duplicates, review_items = deduplicate_cards(cards) + + assert duplicates + assert any(item["reason"] == "same_source" for item in duplicates) + assert any(item["reason"] == "duplicate" for item in review_items) + + +def test_run_deduplication_writes_report(root_path): + subprocess.run(["python3", "scripts/validate_card_passports.py"], cwd=root_path, check=True) + result = subprocess.run( + ["python3", "scripts/run_deduplication.py", "--use-ollama", "off"], + cwd=root_path, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + report = json.loads(DEDUPLICATION_REPORT.read_text(encoding="utf-8")) + assert "duplicate_conflicts" in report + assert report["ollama"]["status"] == "skipped" diff --git a/tests/test_end_to_end_managed_pipeline.py b/tests/test_end_to_end_managed_pipeline.py new file mode 100644 index 0000000..70dbb9b --- /dev/null +++ b/tests/test_end_to_end_managed_pipeline.py @@ -0,0 +1,33 @@ +import json +import subprocess + + +def test_one_command_managed_pipeline_generates_runtime_artifacts(root_path): + result = subprocess.run( + ["python3", "scripts/run_managed_knowledge_factory.py", "--mode", "full", "--use-ollama", "off", "--use-gemini", "off", "--skip-tests"], + cwd=root_path, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + for filename in [ + "governance_state.json", + "card_validation_report.json", + "deduplication_report.json", + "review_queue.json", + "retrieval_qa_results.json", + "acceptance_report.json", + "promotion_report.json", + "release_manifest.json", + ]: + assert (root_path / "publish" / filename).exists() + + promotion = json.loads((root_path / "publish" / "promotion_report.json").read_text(encoding="utf-8")) + acceptance = json.loads((root_path / "publish" / "acceptance_report.json").read_text(encoding="utf-8")) + manifest = json.loads((root_path / "publish" / "release_manifest.json").read_text(encoding="utf-8")) + assert acceptance["acceptance_status"] == "pass" + assert promotion["promotion_status"] == "pass" + assert promotion["production_ready"] is True + assert manifest["production_ready"] == promotion["production_ready"] diff --git a/tests/test_inventory.py b/tests/test_inventory.py new file mode 100644 index 0000000..dd640de --- /dev/null +++ b/tests/test_inventory.py @@ -0,0 +1,36 @@ +import csv + +from notes_to_kb.inventory import FIELDNAMES, SOURCES_FIELDNAMES, build_inventory, source_id_for, write_inventory, write_sources_index + + +def test_inventory_indexes_txt_files_only(raw_dir, workspace): + rows = build_inventory(raw_dir) + output = workspace / "inventory" / "files_index.csv" + write_inventory(rows, output) + + with output.open(encoding="utf-8", newline="") as handle: + csv_rows = list(csv.DictReader(handle)) + + assert len(csv_rows) == 5 + assert list(csv_rows[0].keys()) == FIELDNAMES + assert {row["extension"] for row in csv_rows} == {".txt"} + assert {row["detected_type"] for row in csv_rows} == {"raw_transcript"} + assert all(row["source_id"] == source_id_for(row["base_name"]) for row in csv_rows) + + +def test_sources_index_is_created(raw_dir, workspace): + rows = build_inventory(raw_dir) + output = workspace / "inventory" / "sources_index.csv" + write_sources_index(rows, output) + + with output.open(encoding="utf-8", newline="") as handle: + csv_rows = list(csv.DictReader(handle)) + + assert len(csv_rows) == 5 + assert list(csv_rows[0].keys()) == SOURCES_FIELDNAMES + assert all(row["transcript_path"].endswith(".txt") for row in csv_rows) + + +def test_source_id_is_stable(): + assert source_id_for("Sample_01") == source_id_for("sample_01") + assert source_id_for("sample_01") == source_id_for("sample_01") diff --git a/tests/test_judge.py b/tests/test_judge.py new file mode 100644 index 0000000..8f82cc5 --- /dev/null +++ b/tests/test_judge.py @@ -0,0 +1,153 @@ +import hashlib + +from notes_to_kb.judge import run_judge + + +def write_good_artifacts(workspace): + clean_dir = workspace / "clean_notes" + cards_dir = workspace / "source_cards" + topics_dir = workspace / "knowledge" / "topics" + concepts_dir = workspace / "knowledge" / "concepts" + indexes_dir = workspace / "knowledge" / "indexes" + reports_dir = workspace / "knowledge" / "reports" + for path in (clean_dir, cards_dir, topics_dir, concepts_dir, indexes_dir, reports_dir): + path.mkdir(parents=True, exist_ok=True) + + clean = clean_dir / "abc123.clean.md" + card = cards_dir / "abc123.source_card.md" + topic = topics_dir / "General.md" + concept = concepts_dir / "general.md" + main_index = indexes_dir / "INDEX.md" + sources_index = indexes_dir / "sources_index.csv" + concepts_index = indexes_dir / "concepts_index.csv" + chunk_manifest = workspace / "chunks" / "abc123" / "chunk_manifest.csv" + chunk_manifest.parent.mkdir(parents=True, exist_ok=True) + + clean.write_text( + "\n".join( + [ + "# Clean Note", + "## Metadata", + "- source_id: abc123", + "## Evidence Pointers", + "- chunk_001: evidence", + ] + ), + encoding="utf-8", + ) + card.write_text( + "\n".join( + [ + "# Source Card", + "## Metadata", + "- source_id: abc123", + "- review_required: false", + "## Key Concepts", + "| Concept | Definition | Evidence | Confidence |", + "|---|---|---|---|", + "| x | y | chunk_001 | high |", + "## Procedures / Workflows", + "chunk_001", + "## Human Review", + "- required: false", + ] + ), + encoding="utf-8", + ) + topic.write_text(f"# General\n\n## Source Cards\n- {card.as_posix()}\n", encoding="utf-8") + concept.write_text(f"# General\n\n## Evidence\n- {card.as_posix()}\n", encoding="utf-8") + main_index.write_text(f"# Knowledge Index\n\n## Topics\n- {topic.as_posix()}\n\n## Concepts\n- {concept.as_posix()}\n", encoding="utf-8") + sources_index.write_text( + "source_id,title,source_card_path,clean_note_path,chunk_manifest_path,review_required\n" + f"abc123,Sample,{card.as_posix()},{clean.as_posix()},{chunk_manifest.as_posix()},false\n", + encoding="utf-8", + ) + chunk_manifest.write_text("chunk_id,chunk_path\nchunk_001,chunk_001.txt\n", encoding="utf-8") + concepts_index.write_text( + "concept_id,concept_name,concept_path,source_count,review_required\n" + f"general,General,{concept.as_posix()},1,false\n", + encoding="utf-8", + ) + return clean_dir, cards_dir, topics_dir, concepts_dir, indexes_dir, reports_dir + + +def test_judge_creates_required_reports(workspace): + clean_dir, cards_dir, topics_dir, concepts_dir, indexes_dir, reports_dir = write_good_artifacts(workspace) + + result = run_judge(clean_dir, cards_dir, topics_dir, concepts_dir, indexes_dir, reports_dir) + + assert result.judge_report_path.exists() + assert result.review_queue_path.exists() + assert result.conflicts_path.exists() + assert result.unsupported_claims_path.exists() + assert "# Judge Report" in result.judge_report_path.read_text(encoding="utf-8") + + +def test_judge_detects_think_blocks_missing_evidence_and_review_queue(workspace): + clean_dir, cards_dir, topics_dir, concepts_dir, indexes_dir, reports_dir = write_good_artifacts(workspace) + (clean_dir / "abc123.clean.md").write_text("# Clean Note\nbad\n", encoding="utf-8") + (cards_dir / "abc123.source_card.md").write_text( + "# Source Card\n## Metadata\n- source_id: abc123\n## Key Concepts\n## Procedures / Workflows\n## Human Review\n", + encoding="utf-8", + ) + + result = run_judge(clean_dir, cards_dir, topics_dir, concepts_dir, indexes_dir, reports_dir) + report = result.judge_report_path.read_text(encoding="utf-8") + review_queue = result.review_queue_path.read_text(encoding="utf-8") + + assert "think_block" in report + assert "missing chunk evidence" in report + assert "abc123" in review_queue + + +def test_judge_detects_broken_source_card_link_and_missing_index_target(workspace): + clean_dir, cards_dir, topics_dir, concepts_dir, indexes_dir, reports_dir = write_good_artifacts(workspace) + (topics_dir / "General.md").write_text("# General\n\n## Source Cards\n- missing.source_card.md\n", encoding="utf-8") + (indexes_dir / "concepts_index.csv").write_text( + "concept_id,concept_name,concept_path,source_count,review_required\n" + "bad,Bad,/missing/concept.md,1,false\n", + encoding="utf-8", + ) + + result = run_judge(clean_dir, cards_dir, topics_dir, concepts_dir, indexes_dir, reports_dir) + report = result.judge_report_path.read_text(encoding="utf-8") + + assert result.readiness == "blocked" + assert "broken_source_card_link" in report + assert "missing_index_target" in report + assert "workspace/knowledge/topics/General.md" in report or "General.md" in report + + +def test_judge_accepts_source_card_links_with_spaces_in_path(workspace): + clean_dir, cards_dir, topics_dir, concepts_dir, indexes_dir, reports_dir = write_good_artifacts(workspace) + card = cards_dir / "abc123.source_card.md" + topic = topics_dir / "General.md" + topic.write_text(f"# General\n\n## Source Cards\n- abc123: {card.as_posix()}\n", encoding="utf-8") + + result = run_judge(clean_dir, cards_dir, topics_dir, concepts_dir, indexes_dir, reports_dir) + report = result.judge_report_path.read_text(encoding="utf-8") + + assert "broken_source_card_link" not in report + + +def test_judge_does_not_modify_upstream_artifacts(workspace): + clean_dir, cards_dir, topics_dir, concepts_dir, indexes_dir, reports_dir = write_good_artifacts(workspace) + upstream = [*clean_dir.glob("*"), *cards_dir.glob("*"), *topics_dir.glob("*"), *concepts_dir.glob("*"), *indexes_dir.glob("*")] + before = {path: hashlib.sha256(path.read_bytes()).hexdigest() for path in upstream} + + run_judge(clean_dir, cards_dir, topics_dir, concepts_dir, indexes_dir, reports_dir) + + after = {path: hashlib.sha256(path.read_bytes()).hexdigest() for path in upstream} + assert after == before + + +def test_judge_source_id_mode_limits_clean_notes_and_source_cards(workspace): + clean_dir, cards_dir, topics_dir, concepts_dir, indexes_dir, reports_dir = write_good_artifacts(workspace) + (clean_dir / "other.clean.md").write_text("# Clean Note\nbad\n", encoding="utf-8") + (cards_dir / "other.source_card.md").write_text("# Source Card\nbad\n", encoding="utf-8") + + run_judge(clean_dir, cards_dir, topics_dir, concepts_dir, indexes_dir, reports_dir, source_id="abc123") + report = (reports_dir / "judge_report.md").read_text(encoding="utf-8") + + assert "other.clean.md" not in report + assert "other.source_card.md" not in report diff --git a/tests/test_kb_build.py b/tests/test_kb_build.py new file mode 100644 index 0000000..ad640d4 --- /dev/null +++ b/tests/test_kb_build.py @@ -0,0 +1,144 @@ +import csv +import hashlib + +from notes_to_kb.chunking import build_chunks +from notes_to_kb.clean_note import generate_clean_note +from notes_to_kb.inventory import build_inventory, write_sources_index +from notes_to_kb.kb_build import build_kb +from notes_to_kb.llm_client import MockLLMClient +from notes_to_kb.source_card import generate_source_card + + +def prepare_source_cards(raw_dir, workspace): + rows = build_inventory(raw_dir) + sources_index = workspace / "inventory" / "sources_index.csv" + write_sources_index(rows, sources_index) + chunks_dir = workspace / "chunks" + build_chunks(raw_dir, chunks_dir, chunk_size=80) + prompt_dir = workspace / "prompts" + prompt_dir.mkdir() + clean_prompt = prompt_dir / "clean_note_ru.md" + clean_prompt.write_text("prompt_version: clean_note_ru_v1\n", encoding="utf-8") + source_prompt = prompt_dir / "source_card_ru.md" + source_prompt.write_text("prompt_version: source_card_ru_v1\n", encoding="utf-8") + clean_notes_dir = workspace / "clean_notes" + source_cards_dir = workspace / "source_cards" + kb_sources_index = workspace / "knowledge" / "indexes" / "sources_index.csv" + for row in rows[:2]: + generate_clean_note( + source_id=row.source_id, + sources_index=sources_index, + chunks_root=chunks_dir, + output_dir=clean_notes_dir, + review_queue_path=workspace / "knowledge" / "reports" / "review_queue.md", + client=MockLLMClient(), + prompt_path=clean_prompt, + ) + generate_source_card( + source_id=row.source_id, + sources_index=sources_index, + chunks_root=chunks_dir, + clean_notes_dir=clean_notes_dir, + output_dir=source_cards_dir, + kb_sources_index=kb_sources_index, + review_queue_path=workspace / "knowledge" / "reports" / "review_queue.md", + client=MockLLMClient(), + prompt_path=source_prompt, + overwrite=True, + ) + return chunks_dir, clean_notes_dir, source_cards_dir, kb_sources_index + + +def test_kb_build_creates_topics_concepts_and_indexes(raw_dir, workspace): + _, _, source_cards_dir, kb_sources_index = prepare_source_cards(raw_dir, workspace) + + result = build_kb( + source_cards_dir=source_cards_dir, + source_index_path=kb_sources_index, + topics_dir=workspace / "knowledge" / "topics", + concepts_dir=workspace / "knowledge" / "concepts", + index_path=workspace / "knowledge" / "indexes" / "INDEX.md", + concepts_index_path=workspace / "knowledge" / "indexes" / "concepts_index.csv", + review_queue_path=workspace / "knowledge" / "reports" / "review_queue.md", + overwrite=True, + ) + + assert result.topic_paths + assert result.concept_paths + assert result.index_path.exists() + assert result.concepts_index_path.exists() + topic_text = result.topic_paths[0].read_text(encoding="utf-8") + concept_text = result.concept_paths[0].read_text(encoding="utf-8") + assert "## Source Cards" in topic_text + assert ".source_card.md" in topic_text + assert "## Evidence" in concept_text + assert ".source_card.md" in concept_text + assert "= 18 + + +def test_usage_guide_has_required_sections(): + text = (CHATGPT_ROOT / "KB_USAGE_GUIDE.md").read_text(encoding="utf-8") + + for marker in [ + "Recommended Query Patterns", + "Good Answer Criteria", + "Bad Answer Signals", + ]: + assert marker in text + + +def test_structure_review_has_required_sections(): + text = (PUBLISH_ROOT / "KB_STRUCTURE_REVIEW.md").read_text(encoding="utf-8") + + for marker in [ + "Key Findings", + "Index Issues", + "Recommended Changes", + "Non-goals", + ]: + assert marker in text + + +def test_publish_excludes_txt_files(): + txt_files = [path for path in PUBLISH_ROOT.rglob("*.txt") if path.is_file()] + + assert txt_files == [] + + +def test_publish_excludes_forbidden_artifacts(): + forbidden_name_parts = [ + "embedding", + "embeddings", + "vector", + "scheduler", + "webui", + "web-ui", + ] + + forbidden_paths = [] + for path in PUBLISH_ROOT.rglob("*"): + lower_name = path.name.lower() + if any(part in lower_name for part in forbidden_name_parts): + forbidden_paths.append(path) + + assert forbidden_paths == [] diff --git a/tests/test_kb_structure_revision_pass_2.py b/tests/test_kb_structure_revision_pass_2.py new file mode 100644 index 0000000..f65f217 --- /dev/null +++ b/tests/test_kb_structure_revision_pass_2.py @@ -0,0 +1,139 @@ +import re +import subprocess +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +PUBLISH_ROOT = ROOT / "publish" +PUBLISH_ASSETS_ROOT = ROOT / "publish_assets" +CHATGPT_ROOT = PUBLISH_ROOT / "chatgpt_project" +CHATGPT_ASSETS_ROOT = PUBLISH_ASSETS_ROOT / "chatgpt_project" + + +HELPER_FILES = [ + "CONCEPT_MAP.md", + "WORKFLOW_MAP.md", + "TRACEABILITY_GUIDE.md", +] + + +def test_full_publish_preserves_pass_2_helper_files(): + result = subprocess.run( + ["python3", "scripts/run_publish.py", "--mode", "all"], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + + for filename in HELPER_FILES: + assert (CHATGPT_ASSETS_ROOT / filename).exists() + assert (CHATGPT_ROOT / filename).exists() + + +def test_index_references_pass_2_helper_files(): + text = (CHATGPT_ROOT / "INDEX.md").read_text(encoding="utf-8") + + for filename in [ + "CONCEPT_MAP.md", + "WORKFLOW_MAP.md", + "TRACEABILITY_GUIDE.md", + "KB_USAGE_GUIDE.md", + "SMOKE_QUESTIONS.md", + ]: + assert filename in text + + +def test_usage_guide_references_pass_2_helper_files(): + text = (CHATGPT_ROOT / "KB_USAGE_GUIDE.md").read_text(encoding="utf-8") + + for filename in HELPER_FILES: + assert filename in text + + +def test_smoke_questions_have_at_least_twenty_four_numbered_questions(): + text = (CHATGPT_ROOT / "SMOKE_QUESTIONS.md").read_text(encoding="utf-8") + numbered_questions = re.findall(r"(?m)^\d+\.\s+", text) + + assert len(numbered_questions) >= 24 + assert "source fact" in text + assert "QA recommendation" in text + + +def test_concept_map_has_required_sections(): + text = (CHATGPT_ROOT / "CONCEPT_MAP.md").read_text(encoding="utf-8") + + for marker in [ + "Concept Inventory", + "Package Concepts", + "Concept Relationships", + "Concept Answer Pattern", + "MVP-9.2 Weaknesses Covered", + ]: + assert marker in text + + for concept_id in [ + "ai_systems", + "finance_analytics", + "general_knowledge", + "knowledge_organization", + "power_bi", + "security_workflow", + ]: + assert concept_id in text + + +def test_workflow_map_has_required_sections(): + text = (CHATGPT_ROOT / "WORKFLOW_MAP.md").read_text(encoding="utf-8") + + for marker in [ + "Workflow Inventory", + "Canonical Pipeline", + "QA Gate Map", + "Workflow Extraction Rules", + "MVP-9.2 Weaknesses Covered", + ]: + assert marker in text + + +def test_traceability_guide_has_required_sections(): + text = (CHATGPT_ROOT / "TRACEABILITY_GUIDE.md").read_text(encoding="utf-8") + + for marker in [ + "Traceability Sources", + "Evidence Matrix", + "Answer Grounding Rules", + "Source Traceability Levels", + "MVP-9.2 Weaknesses Covered", + ]: + assert marker in text + + for confidence_level in ["strong", "medium", "weak", "unsupported"]: + assert confidence_level in text + + +def test_publish_excludes_txt_files(): + txt_files = [path for path in PUBLISH_ROOT.rglob("*.txt") if path.is_file()] + + assert txt_files == [] + + +def test_publish_excludes_forbidden_artifacts(): + forbidden_name_parts = [ + "embedding", + "embeddings", + "vector", + "scheduler", + "webui", + "web-ui", + ] + + forbidden_paths = [] + for path in PUBLISH_ROOT.rglob("*"): + lower_name = path.name.lower() + if any(part in lower_name for part in forbidden_name_parts): + forbidden_paths.append(path) + + assert forbidden_paths == [] diff --git a/tests/test_managed_knowledge_system.py b/tests/test_managed_knowledge_system.py new file mode 100644 index 0000000..d00e505 --- /dev/null +++ b/tests/test_managed_knowledge_system.py @@ -0,0 +1,156 @@ +import subprocess +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +COMPACT_ROOT = ROOT / "publish" / "chatgpt_project_compact" + +REQUIRED_GOVERNANCE_FILES = [ + "KB__RELEASE_MANIFEST.md", + "KB__CHANGELOG.md", + "KB__REVIEW_QUEUE.md", + "KB__CARD_SCHEMA.md", + "KB__CONFIDENCE_RULES.md", + "KB__PROMOTION_GATES.md", + "KB__RETRIEVAL_QA.md", + "KB__DEDUPLICATION.md", + "KB__USE_CASE_ROUTING.md", +] + +MANAGED_PIPELINE = ( + "transcript -> chunk -> source card -> concept / workflow / pattern extraction " + "-> grounded synthesis -> publish package -> compact package -> automated smoke QA " + "-> automated acceptance check -> next scope decision -> use-case routing" +) + + +def test_managed_compact_build_generates_governance_files(): + result = subprocess.run( + [ + "zsh", + "-lc", + "python3 scripts/build_chatgpt_compact_kb.py && python3 scripts/build_synthesis_layer.py", + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + + for filename in REQUIRED_GOVERNANCE_FILES: + assert (COMPACT_ROOT / filename).exists(), f"missing governance file: {filename}" + + +def test_canonical_pipeline_does_not_end_at_smoke_qa(): + text = (COMPACT_ROOT / "KB__01_NAVIGATION.md").read_text(encoding="utf-8") + + assert MANAGED_PIPELINE in text + assert "automated acceptance check -> next scope decision -> use-case routing" in text + + +def test_card_schema_has_required_passport_values(): + text = (COMPACT_ROOT / "KB__CARD_SCHEMA.md").read_text(encoding="utf-8") + + for marker in [ + "card_id", + "card_type", + "source_id", + "related_source_ids", + "confidence", + "review_status", + "Source Card", + "Concept Card", + "Workflow Card", + "Pattern Card", + "QA Card", + "Navigation Card", + "Use Case Card", + "approved", + "review_required", + "weak", + "unsupported", + "deprecated", + "duplicate_candidate", + ]: + assert marker in text + + +def test_release_manifest_has_required_governance_fields(): + text = (COMPACT_ROOT / "KB__RELEASE_MANIFEST.md").read_text(encoding="utf-8") + + for marker in [ + "kb_version", + "build_date", + "source_inputs", + "processed_transcripts", + "generated_cards", + "publish_outputs", + "compact_package_outputs", + "smoke_qa_status", + "acceptance_status", + "residual_risks", + "blocked_items", + "next_scope", + "promoted_to_production", + ]: + assert marker in text + + +def test_retrieval_qa_schema_and_promotion_gates_are_automated(): + retrieval = (COMPACT_ROOT / "KB__RETRIEVAL_QA.md").read_text(encoding="utf-8") + gates = (COMPACT_ROOT / "KB__PROMOTION_GATES.md").read_text(encoding="utf-8") + + for marker in [ + "question", + "expected_source", + "actual_source", + "retrieval_status", + "grounding_status", + "confidence_status", + "unsupported_claims", + "final_verdict", + ]: + assert marker in retrieval + + for marker in [ + "Publish gate", + "Boundary gate", + "Preservation gate", + "Consumer QA gate", + "Acceptance gate", + "Promotion gate", + ]: + assert marker in gates + + +def test_promotion_is_blocked_for_unsupported_items(): + manifest = (COMPACT_ROOT / "KB__RELEASE_MANIFEST.md").read_text(encoding="utf-8") + queue = (COMPACT_ROOT / "KB__REVIEW_QUEUE.md").read_text(encoding="utf-8") + + assert "unsupported_items: 0" not in queue + assert "Unsupported items block promotion" in queue + assert "## promoted_to_production\nno" in manifest + + +def test_compact_boundary_excludes_forbidden_artifact_files(): + forbidden_suffixes = {".txt", ".log", ".tmp", ".temp"} + forbidden_name_parts = [ + "embedding", + "embeddings", + "vector_store", + "scheduler", + "webui", + "web-ui", + ] + + forbidden_paths = [] + for path in COMPACT_ROOT.rglob("*"): + lower_name = path.name.lower() + if path.suffix.lower() in forbidden_suffixes: + forbidden_paths.append(path) + elif any(part in lower_name for part in forbidden_name_parts): + forbidden_paths.append(path) + + assert forbidden_paths == [] diff --git a/tests/test_manual_qa_runbook.py b/tests/test_manual_qa_runbook.py new file mode 100644 index 0000000..ed7e3a5 --- /dev/null +++ b/tests/test_manual_qa_runbook.py @@ -0,0 +1,54 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +PUBLISH_ROOT = ROOT / "publish" + + +def test_manual_qa_files_exist(): + required = [ + PUBLISH_ROOT / "MANUAL_QA_RUNBOOK.md", + PUBLISH_ROOT / "MANUAL_QA_RESULTS_TEMPLATE.md", + ] + + for path in required: + assert path.exists(), f"missing manual QA file: {path}" + + +def test_manual_qa_runbook_has_required_sections(): + text = (PUBLISH_ROOT / "MANUAL_QA_RUNBOOK.md").read_text(encoding="utf-8") + + for marker in ["Files to Upload", "Pass Criteria", "Fail Criteria", "Final Verdict"]: + assert marker in text + + +def test_manual_qa_results_template_has_required_sections(): + text = (PUBLISH_ROOT / "MANUAL_QA_RESULTS_TEMPLATE.md").read_text(encoding="utf-8") + + for marker in ["Uploaded Files", "Test Results", "Recommended Next Scope", "Final Verdict"]: + assert marker in text + + +def test_publish_excludes_txt_files(): + txt_files = [path for path in PUBLISH_ROOT.rglob("*.txt") if path.is_file()] + + assert txt_files == [] + + +def test_publish_excludes_forbidden_artifacts(): + forbidden_name_parts = [ + "embedding", + "embeddings", + "vector", + "scheduler", + "webui", + "web-ui", + ] + + forbidden_paths = [] + for path in PUBLISH_ROOT.rglob("*"): + lower_name = path.name.lower() + if any(part in lower_name for part in forbidden_name_parts): + forbidden_paths.append(path) + + assert forbidden_paths == [] diff --git a/tests/test_ollama_client.py b/tests/test_ollama_client.py new file mode 100644 index 0000000..d679ed6 --- /dev/null +++ b/tests/test_ollama_client.py @@ -0,0 +1,305 @@ +import json +import subprocess +from http.client import RemoteDisconnected +from urllib import error + +import pytest + +from notes_to_kb.llm_client import GeminiLLMClient, LLMRequest, MockLLMClient, OllamaLLMClient, build_llm_client + + +class FakeHTTPResponse: + def __init__(self, payload: dict[str, str]) -> None: + self.payload = payload + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback) -> None: + return None + + def read(self) -> bytes: + return json.dumps(self.payload).encode("utf-8") + + def close(self) -> None: + return None + + +def test_build_llm_client_keeps_mock_default(): + client = build_llm_client("mock") + + assert isinstance(client, MockLLMClient) + assert client.provider == "mock" + + +def test_ollama_client_posts_prompt_and_returns_response(monkeypatch): + captured = {} + + def fake_urlopen(http_request, timeout): + captured["url"] = http_request.full_url + captured["timeout"] = timeout + captured["payload"] = json.loads(http_request.data.decode("utf-8")) + return FakeHTTPResponse({"model": "local-model", "response": "## Short Summary\nDone"}) + + monkeypatch.setattr("notes_to_kb.llm_client.request.urlopen", fake_urlopen) + client = OllamaLLMClient("local-model", base_url="http://ollama.local", timeout=7) + + response = client.generate( + LLMRequest( + source_id="abc123", + title="Test Source", + prompt="Return markdown only.", + chunks=[("chunk_001", "Transcript text")], + ) + ) + + assert captured["url"] == "http://ollama.local/api/generate" + assert captured["timeout"] == 7 + assert captured["payload"]["model"] == "local-model" + assert captured["payload"]["stream"] is False + assert captured["payload"]["options"]["temperature"] == 0 + assert "chunk_001" in captured["payload"]["prompt"] + assert "Transcript text" in captured["payload"]["prompt"] + assert response.provider == "ollama" + assert response.model == "local-model" + assert response.text == "## Short Summary\nDone" + + +def test_ollama_client_requires_model(): + with pytest.raises(ValueError, match="--model is required"): + build_llm_client("ollama") + + +def test_ollama_client_rejects_empty_response(monkeypatch): + def fake_urlopen(http_request, timeout): + return FakeHTTPResponse({"model": "local-model", "response": ""}) + + monkeypatch.setattr("notes_to_kb.llm_client.request.urlopen", fake_urlopen) + client = OllamaLLMClient("local-model") + + with pytest.raises(RuntimeError, match="empty response"): + client.generate(LLMRequest(source_id="abc123", title="Test Source", prompt="prompt", chunks=[])) + + +def test_ollama_client_retries_remote_disconnect(monkeypatch): + calls = {"count": 0} + + def fake_urlopen(http_request, timeout): + calls["count"] += 1 + if calls["count"] == 1: + raise RemoteDisconnected("Remote end closed connection without response") + return FakeHTTPResponse({"model": "local-model", "response": "## Short Summary\nRecovered"}) + + monkeypatch.setattr("notes_to_kb.llm_client.request.urlopen", fake_urlopen) + monkeypatch.setattr("notes_to_kb.llm_client.time.sleep", lambda seconds: None) + client = OllamaLLMClient("local-model") + + response = client.generate( + LLMRequest(source_id="abc123", title="Test Source", prompt="prompt", chunks=[]) + ) + + assert calls["count"] == 2 + assert response.text == "## Short Summary\nRecovered" + + +def test_gemini_client_posts_prompt_and_returns_response(monkeypatch): + captured = {} + + def fake_urlopen(http_request, timeout, context=None): + captured["url"] = http_request.full_url + captured["timeout"] = timeout + captured["context"] = context + captured["api_key"] = http_request.headers["X-goog-api-key"] + captured["payload"] = json.loads(http_request.data.decode("utf-8")) + return FakeHTTPResponse( + { + "candidates": [ + {"content": {"parts": [{"text": "## Core Topic\nDone"}]}} + ] + } + ) + + monkeypatch.setattr("notes_to_kb.llm_client.request.urlopen", fake_urlopen) + client = GeminiLLMClient("gemini-test", api_key="test-key", base_url="https://gemini.local", timeout=9) + + response = client.generate( + LLMRequest( + source_id="abc123", + title="Test Source", + prompt="Return markdown only.", + chunks=[("chunk_001", "Transcript text")], + task="source_card", + ) + ) + + assert captured["url"] == "https://gemini.local/v1beta/models/gemini-test:generateContent" + assert captured["timeout"] == 9 + assert captured["context"] is not None + assert captured["api_key"] == "test-key" + assert captured["payload"]["generationConfig"]["temperature"] == 0 + assert captured["payload"]["generationConfig"]["maxOutputTokens"] == 4096 + assert captured["payload"]["generationConfig"]["thinkingConfig"]["thinkingBudget"] == 0 + assert "chunk_001" in captured["payload"]["contents"][0]["parts"][0]["text"] + assert response.provider == "gemini" + assert response.model == "gemini-test" + assert response.text == "## Core Topic\nDone" + + +def test_gemini_client_retries_timeouts(monkeypatch): + calls = {"count": 0} + + def fake_urlopen(http_request, timeout, context=None): + calls["count"] += 1 + if calls["count"] == 1: + raise TimeoutError("read timed out") + return FakeHTTPResponse( + { + "candidates": [ + {"content": {"parts": [{"text": "## Core Topic\nRecovered"}]}} + ] + } + ) + + monkeypatch.setattr("notes_to_kb.llm_client.request.urlopen", fake_urlopen) + monkeypatch.setattr("notes_to_kb.llm_client.time.sleep", lambda seconds: None) + client = GeminiLLMClient("gemini-test", api_key="test-key", base_url="https://gemini.local", timeout=9) + + response = client.generate( + LLMRequest( + source_id="abc123", + title="Test Source", + prompt="Return markdown only.", + chunks=[("chunk_001", "Transcript text")], + task="source_card", + ) + ) + + assert calls["count"] == 2 + assert response.text == "## Core Topic\nRecovered" + + +def test_gemini_client_retries_transient_http_errors(monkeypatch): + calls = {"count": 0} + + def fake_urlopen(http_request, timeout, context=None): + calls["count"] += 1 + if calls["count"] == 1: + raise error.HTTPError( + http_request.full_url, + 503, + "Service Unavailable", + hdrs={}, + fp=FakeHTTPResponse({"error": {"message": "high demand"}}), + ) + return FakeHTTPResponse( + { + "candidates": [ + {"content": {"parts": [{"text": "## Core Topic\nRecovered"}]}} + ] + } + ) + + monkeypatch.setattr("notes_to_kb.llm_client.request.urlopen", fake_urlopen) + monkeypatch.setattr("notes_to_kb.llm_client.time.sleep", lambda seconds: None) + client = GeminiLLMClient("gemini-test", api_key="test-key", base_url="https://gemini.local", timeout=9) + + response = client.generate( + LLMRequest( + source_id="abc123", + title="Test Source", + prompt="Return markdown only.", + chunks=[("chunk_001", "Transcript text")], + task="source_card", + ) + ) + + assert calls["count"] == 2 + assert response.text == "## Core Topic\nRecovered" + + +def test_gemini_client_retries_empty_responses(monkeypatch): + calls = {"count": 0} + + def fake_urlopen(http_request, timeout, context=None): + calls["count"] += 1 + if calls["count"] == 1: + return FakeHTTPResponse({"candidates": [{"finishReason": "MAX_TOKENS"}]}) + return FakeHTTPResponse( + { + "candidates": [ + {"content": {"parts": [{"text": "## Core Topic\nRecovered"}]}} + ] + } + ) + + monkeypatch.setattr("notes_to_kb.llm_client.request.urlopen", fake_urlopen) + monkeypatch.setattr("notes_to_kb.llm_client.time.sleep", lambda seconds: None) + client = GeminiLLMClient("gemini-test", api_key="test-key", base_url="https://gemini.local", timeout=9) + + response = client.generate( + LLMRequest( + source_id="abc123", + title="Test Source", + prompt="Return markdown only.", + chunks=[("chunk_001", "Transcript text")], + task="source_card", + ) + ) + + assert calls["count"] == 2 + assert response.text == "## Core Topic\nRecovered" + + +def test_gemini_client_reports_empty_response_detail(monkeypatch): + def fake_urlopen(http_request, timeout, context=None): + return FakeHTTPResponse({"candidates": [{"finishReason": "SAFETY"}]}) + + monkeypatch.setattr("notes_to_kb.llm_client.request.urlopen", fake_urlopen) + monkeypatch.setattr("notes_to_kb.llm_client.time.sleep", lambda seconds: None) + monkeypatch.setenv("GEMINI_MAX_RETRIES", "0") + client = GeminiLLMClient("gemini-test", api_key="test-key", base_url="https://gemini.local", timeout=9) + + with pytest.raises(RuntimeError, match="finishReason=SAFETY"): + client.generate( + LLMRequest( + source_id="abc123", + title="Test Source", + prompt="Return markdown only.", + chunks=[("chunk_001", "Transcript text")], + task="source_card", + ) + ) + + +def test_gemini_client_requires_api_key(monkeypatch): + monkeypatch.delenv("GEMINI_API_KEY", raising=False) + + with pytest.raises(ValueError, match="GEMINI_API_KEY"): + build_llm_client("gemini", model="gemini-test") + + +def test_clean_note_cli_exposes_ollama_arguments(): + result = subprocess.run( + ["python3", "scripts/run_clean_note.py", "--help"], + check=True, + capture_output=True, + text=True, + ) + + assert "--provider" in result.stdout + assert "--model" in result.stdout + assert "--ollama-base-url" in result.stdout + + +def test_source_card_cli_exposes_ollama_arguments(): + result = subprocess.run( + ["python3", "scripts/run_source_card.py", "--help"], + check=True, + capture_output=True, + text=True, + ) + + assert "--provider" in result.stdout + assert "--model" in result.stdout + assert "--ollama-base-url" in result.stdout + assert "--base-url" in result.stdout diff --git a/tests/test_promotion_gate_runtime.py b/tests/test_promotion_gate_runtime.py new file mode 100644 index 0000000..bb3e155 --- /dev/null +++ b/tests/test_promotion_gate_runtime.py @@ -0,0 +1,94 @@ +import json +import subprocess + +from notes_to_kb.governance import ( + ACCEPTANCE_REPORT, + CARD_VALIDATION_REPORT, + DEDUPLICATION_REPORT, + PROMOTION_REPORT, + RETRIEVAL_QA_RESULTS, + REVIEW_QUEUE_JSON, + write_json, +) + + +def test_acceptance_not_pass_blocks_promotion(): + write_json( + ACCEPTANCE_REPORT, + { + "acceptance_status": "fail", + "blocking_reasons": ["test_blocker"], + }, + ) + + result = subprocess.run( + ["python3", "scripts/run_promotion_gate.py"], + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + report = json.loads(PROMOTION_REPORT.read_text(encoding="utf-8")) + assert report["promotion_status"] == "blocked" + assert report["production_ready"] is False + assert "embeddings" in report["blocked_capabilities"] + + +def run_acceptance_then_promotion(root_path): + subprocess.run(["python3", "scripts/run_acceptance_gate.py"], cwd=root_path, check=True) + subprocess.run(["python3", "scripts/run_promotion_gate.py"], cwd=root_path, check=True) + return json.loads(PROMOTION_REPORT.read_text(encoding="utf-8")) + + +def write_base_passing_inputs(): + write_json(CARD_VALIDATION_REPORT, {"confidence_counts": {"weak": 0, "unsupported": 0}, "schema_invalid_count": 0}) + write_json(DEDUPLICATION_REPORT, {"duplicate_conflicts": 0}) + write_json(RETRIEVAL_QA_RESULTS, {"failed": 0}) + write_json(REVIEW_QUEUE_JSON, {"items": []}) + + +def test_production_ready_false_when_schema_invalid_count_positive(root_path): + write_base_passing_inputs() + write_json(CARD_VALIDATION_REPORT, {"confidence_counts": {"weak": 0, "unsupported": 0}, "schema_invalid_count": 1}) + + report = run_acceptance_then_promotion(root_path) + + assert report["promotion_status"] == "blocked" + assert report["production_ready"] is False + + +def test_production_ready_false_when_unsupported_count_positive(root_path): + write_base_passing_inputs() + write_json(CARD_VALIDATION_REPORT, {"confidence_counts": {"weak": 0, "unsupported": 1}, "schema_invalid_count": 0}) + + report = run_acceptance_then_promotion(root_path) + + assert report["promotion_status"] == "blocked" + assert report["production_ready"] is False + + +def test_production_ready_false_when_high_severity_review_items_exist(root_path): + write_base_passing_inputs() + write_json( + REVIEW_QUEUE_JSON, + { + "items": [ + { + "item_id": "rq_high", + "object_type": "card", + "object_id": "card_high", + "reason": "unsupported", + "severity": "high", + "recommended_action": "request_source", + "source_file": "high.md", + "evidence": [], + } + ] + }, + ) + + report = run_acceptance_then_promotion(root_path) + + assert report["promotion_status"] == "blocked" + assert report["production_ready"] is False diff --git a/tests/test_publish.py b/tests/test_publish.py new file mode 100644 index 0000000..754bdb0 --- /dev/null +++ b/tests/test_publish.py @@ -0,0 +1,141 @@ +import hashlib + +import pytest + +from notes_to_kb.errors import PublishValidationError +from notes_to_kb.publish import PublishInputs, PublishOutputs, run_publish + + +def write_publish_artifacts(workspace, readiness="ready"): + topics_dir = workspace / "knowledge" / "topics" + concepts_dir = workspace / "knowledge" / "concepts" + source_cards_dir = workspace / "source_cards" + indexes_dir = workspace / "knowledge" / "indexes" + reports_dir = workspace / "knowledge" / "reports" + raw_dir = workspace / "input" / "raw" / "transcript_clean" + chunks_dir = workspace / "chunks" / "abc123" + clean_dir = workspace / "clean_notes" + publish_root = workspace / "publish" + for path in (topics_dir, concepts_dir, source_cards_dir, indexes_dir, reports_dir, raw_dir, chunks_dir, clean_dir): + path.mkdir(parents=True, exist_ok=True) + + topic = topics_dir / "General.md" + concept = concepts_dir / "general.md" + card = source_cards_dir / "abc123.source_card.md" + source_index = indexes_dir / "sources_index.csv" + concepts_index = indexes_dir / "concepts_index.csv" + knowledge_index = indexes_dir / "INDEX.md" + judge_report = reports_dir / "judge_report.md" + raw = raw_dir / "abc123.txt" + chunk = chunks_dir / "chunk_001.txt" + manifest = chunks_dir / "chunk_manifest.csv" + clean = clean_dir / "abc123.clean.md" + + card.write_text("# Source Card\n\n## Metadata\n- source_id: abc123\n", encoding="utf-8") + topic.write_text(f"# General\n\n## Source Cards\n- {card.as_posix()}\n", encoding="utf-8") + concept.write_text(f"# General\n\n## Evidence\n- {card.as_posix()}\n", encoding="utf-8") + knowledge_index.write_text(f"# Knowledge Index\n\n## Topics\n- {topic.as_posix()}\n", encoding="utf-8") + source_index.write_text( + "source_id,title,source_card_path,clean_note_path,chunk_manifest_path,review_required\n" + f"abc123,Sample,{card.as_posix()},{clean.as_posix()},{manifest.as_posix()},false\n", + encoding="utf-8", + ) + concepts_index.write_text( + "concept_id,concept_name,concept_path,source_count,review_required\n" + f"general,General,{concept.as_posix()},1,false\n", + encoding="utf-8", + ) + judge_report.write_text( + f"# Judge Report\n\n## Summary\n- checked_artifacts: 4\n- issue_count: 0\n- readiness: {readiness}\n", + encoding="utf-8", + ) + raw.write_text("raw transcript must not be published", encoding="utf-8") + chunk.write_text("chunk", encoding="utf-8") + manifest.write_text("chunk_id,chunk_path\nchunk_001,chunk_001.txt\n", encoding="utf-8") + clean.write_text("# Clean Note\n", encoding="utf-8") + + inputs = PublishInputs(topics_dir, concepts_dir, source_cards_dir, indexes_dir, reports_dir) + outputs = PublishOutputs( + publish_root / "chatgpt_project", + publish_root / "obsidian", + publish_root / "markdown_kb", + ) + upstream = [topic, concept, card, source_index, concepts_index, knowledge_index, judge_report, raw, chunk, manifest, clean] + return inputs, outputs, publish_root, upstream + + +def test_publish_all_creates_required_outputs(workspace): + inputs, outputs, publish_root, _ = write_publish_artifacts(workspace) + + result = run_publish(inputs, outputs, mode="all") + + assert result.judge_readiness == "ready" + assert (publish_root / "chatgpt_project" / "AI_KB_Context_File_v1.0.md").exists() + assert (publish_root / "chatgpt_project" / "INDEX.md").exists() + assert (publish_root / "obsidian" / "topics" / "General.md").exists() + assert (publish_root / "obsidian" / "concepts" / "general.md").exists() + assert (publish_root / "obsidian" / "sources" / "abc123.source_card.md").exists() + assert (publish_root / "obsidian" / "INDEX.md").exists() + assert (publish_root / "markdown_kb" / "full_kb.md").exists() + assert (publish_root / "markdown_kb" / "sources_index.csv").exists() + assert (publish_root / "markdown_kb" / "concepts_index.csv").exists() + + +def test_publish_targeted_mode_only_creates_requested_tree(workspace): + inputs, outputs, publish_root, _ = write_publish_artifacts(workspace) + + run_publish(inputs, outputs, mode="markdown_kb") + + assert (publish_root / "markdown_kb" / "full_kb.md").exists() + assert not (publish_root / "chatgpt_project" / "AI_KB_Context_File_v1.0.md").exists() + assert not (publish_root / "obsidian" / "INDEX.md").exists() + + +def test_publish_preserves_traceability_and_excludes_raw(workspace): + inputs, outputs, publish_root, _ = write_publish_artifacts(workspace) + + run_publish(inputs, outputs, mode="all") + context = (publish_root / "chatgpt_project" / "AI_KB_Context_File_v1.0.md").read_text(encoding="utf-8") + full_kb = (publish_root / "markdown_kb" / "full_kb.md").read_text(encoding="utf-8") + published_files = [path for path in publish_root.glob("**/*") if path.is_file()] + + assert "abc123.source_card.md" in context + assert "sources_index" in full_kb + assert all(path.suffix != ".txt" for path in published_files) + assert "raw transcript must not be published" not in context + assert "raw transcript must not be published" not in full_kb + + +def test_publish_blocks_not_ready_judge_by_default(workspace): + inputs, outputs, _, _ = write_publish_artifacts(workspace, readiness="blocked") + + with pytest.raises(PublishValidationError): + run_publish(inputs, outputs, mode="all") + + +def test_publish_allows_needs_review_judge_by_default(workspace): + inputs, outputs, publish_root, _ = write_publish_artifacts(workspace, readiness="needs_review") + + result = run_publish(inputs, outputs, mode="all") + + assert result.judge_readiness == "needs_review" + assert (publish_root / "chatgpt_project" / "AI_KB_Context_File_v1.0.md").exists() + + +def test_publish_allow_not_ready_override(workspace): + inputs, outputs, publish_root, _ = write_publish_artifacts(workspace, readiness="blocked") + + result = run_publish(inputs, outputs, mode="chatgpt_project", allow_not_ready=True) + + assert result.judge_readiness == "blocked" + assert (publish_root / "chatgpt_project" / "AI_KB_Context_File_v1.0.md").exists() + + +def test_publish_does_not_modify_upstream_artifacts(workspace): + inputs, outputs, _, upstream = write_publish_artifacts(workspace) + before = {path: hashlib.sha256(path.read_bytes()).hexdigest() for path in upstream} + + run_publish(inputs, outputs, mode="all") + + after = {path: hashlib.sha256(path.read_bytes()).hexdigest() for path in upstream} + assert after == before diff --git a/tests/test_publish_preserves_navigation_layer.py b/tests/test_publish_preserves_navigation_layer.py new file mode 100644 index 0000000..a421a31 --- /dev/null +++ b/tests/test_publish_preserves_navigation_layer.py @@ -0,0 +1,96 @@ +import re +import subprocess +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +PUBLISH_ROOT = ROOT / "publish" +CHATGPT_ROOT = PUBLISH_ROOT / "chatgpt_project" + + +def test_full_publish_preserves_navigation_layer(): + result = subprocess.run( + ["python3", "scripts/run_publish.py", "--mode", "all"], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + + required = [ + PUBLISH_ROOT / "KB_STRUCTURE_REVIEW.md", + CHATGPT_ROOT / "INDEX.md", + CHATGPT_ROOT / "SMOKE_QUESTIONS.md", + CHATGPT_ROOT / "KB_USAGE_GUIDE.md", + ] + for path in required: + assert path.exists(), f"missing preserved navigation file: {path}" + + +def test_preserved_index_has_required_sections(): + text = (CHATGPT_ROOT / "INDEX.md").read_text(encoding="utf-8") + + for marker in [ + "Purpose", + "Files to Use", + "How to Ask Questions", + "Decision Map", + "Known Limitations", + ]: + assert marker in text + + +def test_preserved_smoke_questions_have_at_least_eighteen_questions(): + text = (CHATGPT_ROOT / "SMOKE_QUESTIONS.md").read_text(encoding="utf-8") + numbered_questions = re.findall(r"(?m)^\d+\.\s+", text) + + assert len(numbered_questions) >= 18 + + +def test_preserved_usage_guide_has_required_sections(): + text = (CHATGPT_ROOT / "KB_USAGE_GUIDE.md").read_text(encoding="utf-8") + + for marker in [ + "Recommended Query Patterns", + "Good Answer Criteria", + "Bad Answer Signals", + ]: + assert marker in text + + +def test_preserved_structure_review_has_required_sections(): + text = (PUBLISH_ROOT / "KB_STRUCTURE_REVIEW.md").read_text(encoding="utf-8") + + for marker in [ + "Key Findings", + "Recommended Changes", + "Non-goals", + ]: + assert marker in text + + +def test_publish_excludes_txt_files_after_publish(): + txt_files = [path for path in PUBLISH_ROOT.rglob("*.txt") if path.is_file()] + + assert txt_files == [] + + +def test_publish_excludes_forbidden_artifacts_after_publish(): + forbidden_name_parts = [ + "embedding", + "embeddings", + "vector", + "scheduler", + "webui", + "web-ui", + ] + + forbidden_paths = [] + for path in PUBLISH_ROOT.rglob("*"): + lower_name = path.name.lower() + if any(part in lower_name for part in forbidden_name_parts): + forbidden_paths.append(path) + + assert forbidden_paths == [] diff --git a/tests/test_publish_release_package.py b/tests/test_publish_release_package.py new file mode 100644 index 0000000..0f55ce9 --- /dev/null +++ b/tests/test_publish_release_package.py @@ -0,0 +1,60 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +PUBLISH_ROOT = ROOT / "publish" + + +def test_publish_release_docs_exist(): + required = [ + PUBLISH_ROOT / "README.md", + PUBLISH_ROOT / "RELEASE_MANIFEST.md", + PUBLISH_ROOT / "chatgpt_project" / "README.md", + PUBLISH_ROOT / "markdown_kb" / "README.md", + PUBLISH_ROOT / "obsidian" / "README.md", + ] + + for path in required: + assert path.exists(), f"missing release package document: {path}" + + +def test_publish_readme_mentions_core_artifacts(): + text = (PUBLISH_ROOT / "README.md").read_text(encoding="utf-8") + + for expected in [ + "AI_KB_Context_File_v1.0.md", + "INDEX.md", + "full_kb.md", + "sources_index.csv", + "concepts_index.csv", + ]: + assert expected in text + + +def test_publish_package_excludes_txt_files(): + txt_files = [path for path in PUBLISH_ROOT.rglob("*.txt") if path.is_file()] + + assert txt_files == [] + + +def test_publish_package_excludes_forbidden_artifacts(): + forbidden_name_parts = [ + "embedding", + "embeddings", + "vector", + "vector_store", + "scheduler", + "webui", + "web-ui", + ] + forbidden_suffixes = {".db", ".sqlite", ".sqlite3"} + + forbidden_paths = [] + for path in PUBLISH_ROOT.rglob("*"): + lower_name = path.name.lower() + if any(part in lower_name for part in forbidden_name_parts): + forbidden_paths.append(path) + elif path.suffix.lower() in forbidden_suffixes: + forbidden_paths.append(path) + + assert forbidden_paths == [] diff --git a/tests/test_release_manifest_controls.py b/tests/test_release_manifest_controls.py new file mode 100644 index 0000000..e8ddfaa --- /dev/null +++ b/tests/test_release_manifest_controls.py @@ -0,0 +1,50 @@ +import json +import subprocess + +from notes_to_kb.governance import RELEASE_MANIFEST_JSON + + +def test_release_manifest_json_controls_markdown_projection(root_path): + subprocess.run( + ["python3", "scripts/run_managed_knowledge_factory.py", "--mode", "full", "--use-ollama", "off", "--use-gemini", "off", "--skip-tests"], + cwd=root_path, + check=True, + ) + + manifest = json.loads(RELEASE_MANIFEST_JSON.read_text(encoding="utf-8")) + weak_backlog = json.loads((root_path / "publish" / "weak_evidence_backlog.json").read_text(encoding="utf-8")) + audit_snapshot = json.loads((root_path / "publish" / "release_audit_snapshot.json").read_text(encoding="utf-8")) + markdown = (root_path / "publish" / "chatgpt_project_compact" / "KB__RELEASE_MANIFEST.md").read_text(encoding="utf-8") + + for field in [ + "kb_version", + "build_date", + "acceptance_status", + "promotion_status", + "source_count", + "card_count", + "weak_count", + "unsupported_count", + "retrieval_qa_passed", + "retrieval_qa_failed", + "review_required_count", + "production_ready", + "command", + "mode", + "input_paths", + "status", + "blocker_reasons", + "next_action", + ]: + assert field in manifest + + assert "production_ready" in markdown + assert "Weak evidence meaning" in markdown + assert "Gemini skipped is not a blocker" in markdown + assert "Ollama may be available" in markdown + assert weak_backlog["weak_count"] == manifest["weak_count"] + assert len(weak_backlog["top_20_weakest_cards"]) <= 20 + assert "git_status" in audit_snapshot + assert audit_snapshot["production_ready"] == manifest["production_ready"] + + assert ("## promoted_to_production\nyes" in markdown) == manifest["production_ready"] diff --git a/tests/test_retrieval_qa_runtime.py b/tests/test_retrieval_qa_runtime.py new file mode 100644 index 0000000..883e1c5 --- /dev/null +++ b/tests/test_retrieval_qa_runtime.py @@ -0,0 +1,40 @@ +import json +import subprocess + +from scripts.run_retrieval_qa import evaluate_case +from notes_to_kb.governance import RETRIEVAL_QA_RESULTS + + +def test_missing_expected_source_fails_retrieval_case(): + result = evaluate_case( + { + "question": "Missing file?", + "expected_source": "KB__DOES_NOT_EXIST.md", + "expected_section": "Missing", + "required_confidence": "medium", + "must_find": True, + } + ) + + assert result["retrieval_status"] == "fail" + assert result["confidence_status"] == "fail" + assert result["final_verdict"] == "fail" + + +def test_run_retrieval_qa_writes_runtime_results(root_path): + subprocess.run(["python3", "scripts/build_chatgpt_compact_kb.py"], cwd=root_path, check=True) + subprocess.run(["python3", "scripts/build_synthesis_layer.py"], cwd=root_path, check=True) + result = subprocess.run( + ["python3", "scripts/run_retrieval_qa.py", "--use-ollama", "off", "--use-gemini", "off"], + cwd=root_path, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + report = json.loads(RETRIEVAL_QA_RESULTS.read_text(encoding="utf-8")) + assert report["case_count"] >= 3 + assert report["ollama"]["status"] == "skipped" + assert report["gemini"]["status"] == "skipped" + assert all("expected_source" in item for item in report["results"]) diff --git a/tests/test_runtime_confidence_controls.py b/tests/test_runtime_confidence_controls.py new file mode 100644 index 0000000..3df5658 --- /dev/null +++ b/tests/test_runtime_confidence_controls.py @@ -0,0 +1,20 @@ +from notes_to_kb.governance import CONFIDENCE_ORDER, confidence_from_evidence + + +def test_confidence_thresholds_are_deterministic(): + assert confidence_from_evidence(5) == "strong" + assert confidence_from_evidence(2) == "medium" + assert confidence_from_evidence(1) == "weak" + assert confidence_from_evidence(0) == "unsupported" + + +def test_schema_gap_or_conflict_degrades_confidence(): + assert confidence_from_evidence(5, schema_valid=False) == "weak" + assert confidence_from_evidence(5, has_conflict=True) == "weak" + assert confidence_from_evidence(5, deprecated=True) == "weak" + assert confidence_from_evidence(5, missing_source=True) == "unsupported" + + +def test_confidence_order_blocks_unsupported_from_operational_thresholds(): + assert CONFIDENCE_ORDER["unsupported"] < CONFIDENCE_ORDER["medium"] + assert CONFIDENCE_ORDER["weak"] < CONFIDENCE_ORDER["medium"] diff --git a/tests/test_search_cli_mvp_10.py b/tests/test_search_cli_mvp_10.py new file mode 100644 index 0000000..4b0ffad --- /dev/null +++ b/tests/test_search_cli_mvp_10.py @@ -0,0 +1,94 @@ +import json +import subprocess +from pathlib import Path + +import pytest + +from notes_to_kb.search import search_file, search_text + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "search_kb.py" +FULL_KB = ROOT / "publish" / "markdown_kb" / "full_kb.md" + + +def test_search_text_finds_case_insensitive_snippet(): + results = search_text("Alpha\nKnowledge Base pipeline\nOmega", query="knowledge base", limit=5) + + assert len(results) == 1 + assert results[0].line_number == 2 + assert "Knowledge Base" in results[0].snippet + + +def test_search_text_respects_limit(): + results = search_text("KB\nKB\nKB", query="KB", limit=2) + + assert len(results) == 2 + + +def test_search_text_rejects_empty_query(): + with pytest.raises(ValueError, match="query must not be empty"): + search_text("content", query=" ") + + +def test_search_file_rejects_missing_file(tmp_path): + with pytest.raises(FileNotFoundError, match="search file not found"): + search_file(tmp_path / "missing.md", query="KB") + + +def test_cli_text_output_returns_known_query(): + result = subprocess.run( + ["python3", str(SCRIPT), "Knowledge", "--limit", "2"], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + assert "query=Knowledge" in result.stdout + assert "result_count=" in result.stdout + assert "Knowledge" in result.stdout + + +def test_cli_json_output_is_valid_and_limited(): + result = subprocess.run( + ["python3", str(SCRIPT), "Knowledge", "--limit", "1", "--json"], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + payload = json.loads(result.stdout) + assert payload["query"] == "Knowledge" + assert payload["path"] == FULL_KB.as_posix() + assert len(payload["results"]) <= 1 + assert {"line_number", "snippet"} <= set(payload["results"][0]) + + +def test_cli_rejects_missing_file(tmp_path): + result = subprocess.run( + ["python3", str(SCRIPT), "Knowledge", "--path", str(tmp_path / "missing.md")], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert "search file not found" in result.stderr + + +def test_cli_rejects_empty_query(): + result = subprocess.run( + ["python3", str(SCRIPT), " ", "--path", str(FULL_KB)], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert "query must not be empty" in result.stderr diff --git a/tests/test_source_card.py b/tests/test_source_card.py new file mode 100644 index 0000000..b98d904 --- /dev/null +++ b/tests/test_source_card.py @@ -0,0 +1,389 @@ +import csv +import hashlib + +from notes_to_kb.chunking import build_chunks +from notes_to_kb.clean_note import generate_clean_note +from notes_to_kb.inventory import build_inventory, write_sources_index +from notes_to_kb.llm_client import MockLLMClient +from notes_to_kb.source_card import ( + clean_model_body, + classify_body_quality, + generate_all_source_cards, + generate_source_card, + repair_required_sections, + select_sources_for_source_card_generation, + validate_body, +) + + +class CountingMockLLMClient(MockLLMClient): + def __init__(self): + self.calls = [] + + def generate(self, request): + self.calls.append(request.source_id) + return super().generate(request) + + +def prepare_clean_note(raw_dir, workspace): + rows = build_inventory(raw_dir) + sources_index = workspace / "inventory" / "sources_index.csv" + write_sources_index(rows, sources_index) + chunks_dir = workspace / "chunks" + build_chunks(raw_dir, chunks_dir, chunk_size=80) + prompt_dir = workspace / "prompts" + prompt_dir.mkdir() + clean_prompt = prompt_dir / "clean_note_ru.md" + clean_prompt.write_text("prompt_version: clean_note_ru_v1\n", encoding="utf-8") + source_id = rows[0].source_id + generate_clean_note( + source_id=source_id, + sources_index=sources_index, + chunks_root=chunks_dir, + output_dir=workspace / "clean_notes", + review_queue_path=workspace / "knowledge" / "reports" / "review_queue.md", + client=MockLLMClient(), + prompt_path=clean_prompt, + ) + source_prompt = prompt_dir / "source_card_ru.md" + source_prompt.write_text("prompt_version: source_card_ru_v1\n", encoding="utf-8") + return source_id, sources_index, chunks_dir, workspace / "clean_notes", source_prompt + + +def test_source_card_is_created_with_metadata_evidence_and_index(raw_dir, workspace): + source_id, sources_index, chunks_dir, clean_notes_dir, prompt = prepare_clean_note(raw_dir, workspace) + + result = generate_source_card( + source_id=source_id, + sources_index=sources_index, + chunks_root=chunks_dir, + clean_notes_dir=clean_notes_dir, + output_dir=workspace / "source_cards", + kb_sources_index=workspace / "knowledge" / "indexes" / "sources_index.csv", + review_queue_path=workspace / "knowledge" / "reports" / "review_queue.md", + client=MockLLMClient(), + prompt_path=prompt, + ) + + card = result.output_path.read_text(encoding="utf-8") + assert "# Source Card" in card + assert f"- source_id: {source_id}" in card + assert "- processing_mode: mock" in card + assert "- model: mock-source-card-v1" in card + assert "- prompt_version: source_card_ru_v1" in card + assert "- confidence:" in card + assert "| Concept | Definition | Evidence | Confidence |" in card + assert "| Procedure | Steps | Evidence | Caveats |" in card + assert "| Rule | When to use | Evidence | Risk |" in card + assert "chunk_001" in card + assert "