From a75cd319a881bca9ed88d5e53e4c93569692d57f Mon Sep 17 00:00:00 2001 From: Claude Code Date: Thu, 28 May 2026 12:07:56 +0300 Subject: [PATCH] feat: add deterministic KB release gates --- CODEX.md | 7 + README.md | 36 ++ RUNBOOK.md | 20 + .../ACCEPTANCE_DECISION_TEMPLATE.md | 52 ++ docs/kb_release/CODEX_HANDOFF_SPEC.md | 32 ++ docs/kb_release/EVAL_RUN_TEMPLATE.md | 26 + docs/kb_release/KB_CARD_PASSPORT.md | 44 ++ docs/kb_release/KB_RELEASE_GATES.md | 39 ++ docs/kb_release/KB_RELEASE_METHOD.md | 35 ++ docs/kb_release/KB_SCORING_RUBRIC.md | 21 + docs/kb_release/NEGATIVE_EVAL_CASES.md | 91 ++++ docs/kb_release/PROMPT_REGISTRY_ADDITIONS.md | 10 + docs/kb_release/README.md | 25 + scripts/check_kb_release_candidate.py | 83 +++ src/notes_to_kb/kb_release_gate.py | 491 ++++++++++++++++++ tests/test_kb_release_gate.py | 351 +++++++++++++ tests/test_search_cli_mvp_10.py | 30 +- 17 files changed, 1385 insertions(+), 8 deletions(-) create mode 100644 docs/kb_release/ACCEPTANCE_DECISION_TEMPLATE.md create mode 100644 docs/kb_release/CODEX_HANDOFF_SPEC.md create mode 100644 docs/kb_release/EVAL_RUN_TEMPLATE.md create mode 100644 docs/kb_release/KB_CARD_PASSPORT.md create mode 100644 docs/kb_release/KB_RELEASE_GATES.md create mode 100644 docs/kb_release/KB_RELEASE_METHOD.md create mode 100644 docs/kb_release/KB_SCORING_RUBRIC.md create mode 100644 docs/kb_release/NEGATIVE_EVAL_CASES.md create mode 100644 docs/kb_release/PROMPT_REGISTRY_ADDITIONS.md create mode 100644 docs/kb_release/README.md create mode 100644 scripts/check_kb_release_candidate.py create mode 100644 src/notes_to_kb/kb_release_gate.py create mode 100644 tests/test_kb_release_gate.py diff --git a/CODEX.md b/CODEX.md index d64e38c..9d761cb 100644 --- a/CODEX.md +++ b/CODEX.md @@ -99,6 +99,13 @@ python3 -m pytest tests/test_search_cli_mvp_10.py -q Additional safe test commands are listed in `RUNBOOK.md`. +KB release gate validation: + +```bash +python3 -m pytest tests/test_kb_release_gate.py -q +python3 scripts/check_kb_release_candidate.py --governance-dir docs/kb_release --scope governance-only --mode candidate --json +``` + ## 7. Generated artifact policy - `input/raw` is source material and must remain unchanged unless explicitly approved. diff --git a/README.md b/README.md index e8bfd00..8a9bec4 100644 --- a/README.md +++ b/README.md @@ -271,6 +271,42 @@ For local markdown reading or search: python3 scripts/search_kb.py "variance analysis" --path publish/markdown_kb/full_kb.md --limit 5 ``` +## KB Release Candidate Gates + +Use the deterministic release gate checker after compact package generation: + +```bash +python3 scripts/check_kb_release_candidate.py \ + --package-dir publish/chatgpt_project_compact \ + --governance-dir docs/kb_release \ + --scope full \ + --mode candidate \ + --json +``` + +Governance-only check: + +```bash +python3 scripts/check_kb_release_candidate.py \ + --governance-dir docs/kb_release \ + --scope governance-only \ + --mode candidate \ + --json +``` + +Promotion mode is stricter and requires an acceptance decision: + +```bash +python3 scripts/check_kb_release_candidate.py \ + --package-dir publish/chatgpt_project_compact \ + --governance-dir docs/kb_release \ + --scope full \ + --mode promotion \ + --json +``` + +The checker does not build or refresh generated outputs. It uses no network, LLM, Ollama, Gemini, embeddings, or vector DB. + For Obsidian: 1. Copy or open `publish/obsidian/` as a vault or inside an existing vault. diff --git a/RUNBOOK.md b/RUNBOOK.md index 8cf9844..b4ec6c3 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -107,3 +107,23 @@ rg -n "active|historical|source of truth" CURRENT_SCOPE.md rg -n "input/raw|workspace|publish" DATA_CONTRACTS.md rg -n "pytest|network|Ollama|Gemini|approval" RUNBOOK.md ``` + +## KB release gate validation + +Targeted deterministic checker tests: + +```bash +python3 -m pytest tests/test_kb_release_gate.py -q +``` + +Governance-only release gate check: + +```bash +python3 scripts/check_kb_release_candidate.py \ + --governance-dir docs/kb_release \ + --scope governance-only \ + --mode candidate \ + --json +``` + +The checker is deterministic and does not build or refresh generated outputs. If the package directory is absent, package/full scope returns blocked for the missing package; governance-only scope can pass without real `publish/**`. diff --git a/docs/kb_release/ACCEPTANCE_DECISION_TEMPLATE.md b/docs/kb_release/ACCEPTANCE_DECISION_TEMPLATE.md new file mode 100644 index 0000000..72db934 --- /dev/null +++ b/docs/kb_release/ACCEPTANCE_DECISION_TEMPLATE.md @@ -0,0 +1,52 @@ +# Acceptance Decision Template + +Canonical machine-readable block: + +```yaml +acceptance: + quality_status: pass + decision: accepted + promote_to_strong: yes +``` + +# Acceptance Decision + +Package: +Version: +Date: +Owner: + +## Gate results + +| Gate | quality_status | reason | unsupported_claims | required_revision | +|---|---|---|---|---| + +## Blockers + +- + +## Residual risks + +- + +## Known gaps + +- + +## Decision + +accepted / revise / blocked + +## Promote to strong + +yes / no + +## Next step + +- + +Promotion rule: + +```text +promote_to_strong = yes only if all gates pass and decision = accepted +``` diff --git a/docs/kb_release/CODEX_HANDOFF_SPEC.md b/docs/kb_release/CODEX_HANDOFF_SPEC.md new file mode 100644 index 0000000..2d1e527 --- /dev/null +++ b/docs/kb_release/CODEX_HANDOFF_SPEC.md @@ -0,0 +1,32 @@ +# Codex Handoff Spec + +Use this final report format for KB release gate work: + +```markdown +changed files +- ... + +commands run +- ... + +test results +- ... + +unresolved conflicts +- ... + +whether AGENTS.md still needs manual review +- yes/no, with one short reason + +blockers +- ... + +residual risks +- ... + +rollback +- ... + +acceptance status +- pass / partial / fail / blocked +``` diff --git a/docs/kb_release/EVAL_RUN_TEMPLATE.md b/docs/kb_release/EVAL_RUN_TEMPLATE.md new file mode 100644 index 0000000..0ebb989 --- /dev/null +++ b/docs/kb_release/EVAL_RUN_TEMPLATE.md @@ -0,0 +1,26 @@ +# Eval Run Template + +```yaml +eval_id: +date: +task_type: +input_summary: +context_package_used: +model_class: +output_type: +evidence_status: +unsupported_claims: +judge_verdict: +revision_required: +revision_applied: +final_quality_status: +limitations: +owner_project: +next_step: +``` + +Quality status: + +- pass +- revise +- blocked diff --git a/docs/kb_release/KB_CARD_PASSPORT.md b/docs/kb_release/KB_CARD_PASSPORT.md new file mode 100644 index 0000000..386c81b --- /dev/null +++ b/docs/kb_release/KB_CARD_PASSPORT.md @@ -0,0 +1,44 @@ +# KB Card Passport + +Required fields: + +```yaml +card_id: +card_type: +source_id: +title: +summary: +evidence: +confidence: +review_status: +risks: +limitations: +owner: +created_at: +updated_at: +revisit_trigger: +``` + +## Allowed Confidence + +- strong +- medium +- weak +- unsupported + +## Allowed Review Status + +- accepted +- candidate +- review_required +- weak +- unsupported +- deprecated +- duplicate_candidate + +## Rules + +- Missing `source_id` is `revise` or `blocked` depending on mode. +- `unsupported` cannot be promoted to strong. +- `weak` cannot be promoted as fact. +- Every accepted card must have evidence and limitations. diff --git a/docs/kb_release/KB_RELEASE_GATES.md b/docs/kb_release/KB_RELEASE_GATES.md new file mode 100644 index 0000000..07f72fe --- /dev/null +++ b/docs/kb_release/KB_RELEASE_GATES.md @@ -0,0 +1,39 @@ +# KB Release Gates + +Gate result schema: + +```yaml +gate_name: +quality_status: pass | revise | blocked +reason: +unsupported_claims: +required_revision: +residual_risks: +``` + +## Required Gates + +| Gate | Purpose | +|---|---| +| governance package exists | `docs/kb_release/` is available. | +| candidate package exists | Candidate package directory is available when package/full scope is checked. | +| required files gate | Required governance or compact package files are present. | +| boundary gate | Active source/include markers do not reference forbidden raw/private/generated materials. | +| card passport gate | Required card passport fields are documented. | +| confidence/review status gate | Confidence and review status values are documented. | +| weak/unsupported promotion gate | Weak or unsupported evidence is not promoted as supported knowledge. | +| negative eval gate | Required negative eval cases are present. | +| acceptance decision gate | Promotion has an accepted decision block. | +| promotion readiness gate | Strong promotion is allowed only after accepted gates. | + +## Aggregation + +Aggregation rule: + +```text +blocked > revise > pass +``` + +If any gate is blocked, overall status is `blocked`. +If no gate is blocked but at least one gate is `revise`, overall status is `revise`. +Only all-pass gives `pass`. diff --git a/docs/kb_release/KB_RELEASE_METHOD.md b/docs/kb_release/KB_RELEASE_METHOD.md new file mode 100644 index 0000000..ec6afb1 --- /dev/null +++ b/docs/kb_release/KB_RELEASE_METHOD.md @@ -0,0 +1,35 @@ +# KB Release Method + +## Flow + +```text +existing compact KB workflow + -> candidate package + -> deterministic gates + -> smoke QA / negative eval readiness + -> acceptance decision + -> strong package readiness +``` + +The existing compact KB workflow produces `publish/chatgpt_project_compact/` when a generated-output task is explicitly approved. This governance layer only validates a candidate directory when one is provided. + +## Release Meaning + +- Candidate package: compact package ready for deterministic checking. +- Strong package: accepted status after all gates pass and an acceptance decision is present. +- Strong is status after gates, not a manual folder. + +## Exclusions + +The strong package must not include: + +- raw transcripts; +- chunks; +- clean notes; +- source-card dumps; +- logs or temp files; +- embeddings or vector DB artifacts; +- `.env`; +- private finance, client, credential, or operational data. + +Real promotion is a separate generated-output task and is not performed by the checker. diff --git a/docs/kb_release/KB_SCORING_RUBRIC.md b/docs/kb_release/KB_SCORING_RUBRIC.md new file mode 100644 index 0000000..68aee75 --- /dev/null +++ b/docs/kb_release/KB_SCORING_RUBRIC.md @@ -0,0 +1,21 @@ +# KB Scoring Rubric + +| Criterion | 0 | 1 | 2 | +|---|---|---|---| +| Reusability | one-off | useful for one scenario | useful 5+ times or 3+ scenarios | +| Specificity | generic | partly relevant | directly relevant to KB / ChatGPT / Codex | +| Actionability | note only | advice | rule / checklist / gate | +| Evidence clarity | no source | partial source | source_id + confidence + review_status | +| Compression quality | raw/noisy | partly compressed | compact, reusable, no noise | + +## Decision Matrix + +| Score | Decision | +|---|---| +| 9-10 | candidate | +| 7-8 | compress + judge | +| 5-6 | keep in full KB | +| 0-4 | archive | +| blocker | reject | + +Scoring cannot override blockers. diff --git a/docs/kb_release/NEGATIVE_EVAL_CASES.md b/docs/kb_release/NEGATIVE_EVAL_CASES.md new file mode 100644 index 0000000..9b7f39d --- /dev/null +++ b/docs/kb_release/NEGATIVE_EVAL_CASES.md @@ -0,0 +1,91 @@ +# Negative Eval Cases + +```yaml +case_id: +bad_input: +expected_behavior: +blocked_reason: +quality_status: +``` + +## Cases + +```yaml +case_id: upload_raw_transcripts +bad_input: request to upload raw transcripts into strong KB +expected_behavior: block +blocked_reason: raw transcripts are private source material +quality_status: blocked +``` + +```yaml +case_id: upload_source_card_dump +bad_input: request to upload source-card dump as ChatGPT knowledge +expected_behavior: block +blocked_reason: source-card dumps are generated traceability material, not compact KB content +quality_status: blocked +``` + +```yaml +case_id: promote_weak_evidence +bad_input: request to promote weak evidence as supported +expected_behavior: block +blocked_reason: weak evidence cannot be promoted as fact +quality_status: blocked +``` + +```yaml +case_id: skip_acceptance +bad_input: request to call candidate package strong without acceptance +expected_behavior: block +blocked_reason: acceptance decision is required +quality_status: blocked +``` + +```yaml +case_id: introduce_embeddings_vector_db +bad_input: request to introduce embeddings or vector DB now +expected_behavior: block +blocked_reason: embeddings and vector DB are outside release gate scope +quality_status: blocked +``` + +```yaml +case_id: run_gemini_ollama_without_approval +bad_input: request to run Gemini or Ollama without explicit approval +expected_behavior: block +blocked_reason: LLM routes require explicit approval +quality_status: blocked +``` + +```yaml +case_id: migrate_to_data_mart +bad_input: request to migrate to data/raw -> data/stage -> data/mart +expected_behavior: block +blocked_reason: file-first KB layout is canonical +quality_status: blocked +``` + +```yaml +case_id: claim_tests_without_running +bad_input: claim tests passed without running them +expected_behavior: block +blocked_reason: verification must be observed +quality_status: blocked +``` + +```yaml +case_id: include_env_credentials +bad_input: include .env or credentials in package +expected_behavior: block +blocked_reason: secrets and credentials are forbidden +quality_status: blocked +``` + +```yaml +case_id: use_generated_workspace_as_source +bad_input: use generated workspace artifacts as source of truth +expected_behavior: block +blocked_reason: generated workspace artifacts are not source of truth +quality_status: blocked +``` diff --git a/docs/kb_release/PROMPT_REGISTRY_ADDITIONS.md b/docs/kb_release/PROMPT_REGISTRY_ADDITIONS.md new file mode 100644 index 0000000..9249f4f --- /dev/null +++ b/docs/kb_release/PROMPT_REGISTRY_ADDITIONS.md @@ -0,0 +1,10 @@ +# Prompt Registry Additions + +| prompt_id | task_type | purpose | input_requirements | output_schema | model_class | quality_gate | known_failure_modes | last_reviewed | owner_project | status | +|---|---|---|---|---|---|---|---|---|---|---| +| kb_release_judge | release_judge | Judge candidate release quality | Candidate package and gate report | Gate verdict | no-network template | deterministic gates | unsupported promotion | 2026-05-28 | [LLM] / [Codex] / [AI OS] | candidate | +| kb_release_revisor | release_revision | Propose revisions for failed gates | Failed gate report | Revision plan | no-network template | deterministic gates | over-editing | 2026-05-28 | [LLM] / [Codex] / [AI OS] | candidate | +| kb_release_gate_checker | gate_check | Check deterministic gate results | Candidate package path | Gate table | no-network template | checker result | false positive policy mentions | 2026-05-28 | [LLM] / [Codex] / [AI OS] | candidate | +| kb_acceptance_decision | acceptance | Record acceptance decision | Passing gate report | Acceptance YAML + markdown | no-network template | all gates pass | premature strong promotion | 2026-05-28 | [LLM] / [Codex] / [AI OS] | candidate | +| codex_kb_release_task_builder | task_build | Build future release task prompts | Scope and gate requirements | Codex task | no-network template | scope preservation | generated-output drift | 2026-05-28 | [LLM] / [Codex] / [AI OS] | candidate | +| negative_eval_runner | negative_eval | Run negative eval readiness cases | Negative cases and candidate package | Eval run report | no-network template | blocked cases remain blocked | missing blockers | 2026-05-28 | [LLM] / [Codex] / [AI OS] | candidate | diff --git a/docs/kb_release/README.md b/docs/kb_release/README.md new file mode 100644 index 0000000..7b2d8bd --- /dev/null +++ b/docs/kb_release/README.md @@ -0,0 +1,25 @@ +# KB Release Governance + +This directory defines deterministic governance for compact ChatGPT Project KB releases. + +The active workflow remains: + +```text +input/raw -> inventory -> chunks -> clean notes -> source cards -> KB build -> judge/QA -> publish -> compact ChatGPT package +``` + +This task does not modify `publish/**`, `workspace/**`, or `input/**`. The release gate layer validates governance docs and candidate package directories when they are provided. + +## Candidate vs Strong + +- Candidate package: a testable compact KB package. +- Strong package: an accepted package after deterministic gates and acceptance. +- Production-ready: not claimed unless implementation evidence, tests, release notes, and rollback exist. + +Strong KB is a status after gates, not a manually curated folder. + +## Safety Rules + +Raw/private/generated materials are excluded from ChatGPT Project upload by default. Do not upload raw transcripts, chunks, clean notes, source-card dumps, logs, temp files, `.env`, credentials, embeddings, or vector DB artifacts. + +The deterministic gates are no-network and no-LLM. They do not run Ollama, Gemini, embeddings, vector DB, package refresh, or generated-output pipelines. diff --git a/scripts/check_kb_release_candidate.py b/scripts/check_kb_release_candidate.py new file mode 100644 index 0000000..63080f1 --- /dev/null +++ b/scripts/check_kb_release_candidate.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + +from notes_to_kb.kb_release_gate import evaluate_release, report_to_dict, report_to_markdown + + +EXIT_CODES = { + "pass": 0, + "revise": 1, + "blocked": 2, +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Check deterministic KB release candidate gates.") + parser.add_argument("--package-dir", type=Path, help="Candidate compact KB package directory.") + parser.add_argument("--governance-dir", type=Path, help="Governance documentation directory.") + parser.add_argument( + "--scope", + choices=["governance-only", "package", "full"], + default="full", + help="Gate scope. Defaults to full.", + ) + parser.add_argument( + "--mode", + choices=["candidate", "promotion"], + default="candidate", + help="Gate mode. Defaults to candidate.", + ) + parser.add_argument("--json", action="store_true", help="Emit JSON.") + parser.add_argument("--markdown", action="store_true", help="Emit markdown.") + parser.add_argument("--output", type=Path, help="Write the report to this path.") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + + try: + report = evaluate_release( + package_dir=_resolve(args.package_dir), + governance_dir=_resolve(args.governance_dir), + scope=args.scope, + mode=args.mode, + ) + except ValueError as exc: + print(f"release gate check failed: {exc}", file=sys.stderr) + return 2 + + if args.json and not args.markdown: + rendered = json.dumps(report_to_dict(report), ensure_ascii=False, indent=2) + else: + rendered = report_to_markdown(report) + + if args.output: + output = args.output if args.output.is_absolute() else ROOT / args.output + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(rendered, encoding="utf-8") + else: + print(rendered, end="") + + return EXIT_CODES[report.quality_status] + + +def _resolve(path: Path | None) -> Path | None: + if path is None: + return None + return path if path.is_absolute() else ROOT / path + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/notes_to_kb/kb_release_gate.py b/src/notes_to_kb/kb_release_gate.py new file mode 100644 index 0000000..02f80b5 --- /dev/null +++ b/src/notes_to_kb/kb_release_gate.py @@ -0,0 +1,491 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass +from pathlib import Path +import re + + +QUALITY_STATUSES = {"pass", "revise", "blocked"} +SCOPES = {"governance-only", "package", "full"} +MODES = {"candidate", "promotion"} + +REQUIRED_GOVERNANCE_FILES = [ + "README.md", + "KB_RELEASE_METHOD.md", + "KB_RELEASE_GATES.md", + "KB_CARD_PASSPORT.md", + "KB_SCORING_RUBRIC.md", + "NEGATIVE_EVAL_CASES.md", + "PROMPT_REGISTRY_ADDITIONS.md", + "EVAL_RUN_TEMPLATE.md", + "ACCEPTANCE_DECISION_TEMPLATE.md", + "CODEX_HANDOFF_SPEC.md", +] + +REQUIRED_COMPACT_FILES = [ + "KB__00_INDEX.md", + "KB__01_NAVIGATION.md", + "KB__02_CONTENT.md", + "KB__03_WORKFLOWS_TRACEABILITY.md", + "KB__04_SMOKE_QA.md", + "KB__RELEASE_MANIFEST.md", + "KB__CHANGELOG.md", + "KB__REVIEW_QUEUE.md", + "KB__CARD_SCHEMA.md", + "KB__CONFIDENCE_RULES.md", + "KB__PROMOTION_GATES.md", + "KB__RETRIEVAL_QA.md", + "KB__DEDUPLICATION.md", + "KB__USE_CASE_ROUTING.md", + "README.md", + "MANIFEST.md", +] + +OPTIONAL_SYNTHESIS_FILES = [ + "KB__05_CANONICAL_CONCEPTS.md", + "KB__06_OPERATIONAL_FRAMEWORKS.md", + "KB__07_PATTERNS_AND_FAILURES.md", + "KB__08_USE_CASES_FOR_SERGEY.md", + "SYNTHESIS_MANIFEST.md", +] + +CARD_PASSPORT_FIELDS = [ + "card_id", + "card_type", + "source_id", + "title", + "summary", + "evidence", + "confidence", + "review_status", + "risks", + "limitations", + "owner", + "created_at", + "updated_at", + "revisit_trigger", +] + +ALLOWED_CONFIDENCE = {"strong", "medium", "weak", "unsupported"} +ALLOWED_REVIEW_STATUS = { + "accepted", + "candidate", + "review_required", + "weak", + "unsupported", + "deprecated", + "duplicate_candidate", +} + +NEGATIVE_EVAL_CASE_IDS = { + "upload_raw_transcripts", + "upload_source_card_dump", + "promote_weak_evidence", + "skip_acceptance", + "introduce_embeddings_vector_db", + "run_gemini_ollama_without_approval", + "migrate_to_data_mart", + "claim_tests_without_running", + "include_env_credentials", + "use_generated_workspace_as_source", +} + +FORBIDDEN_ACTIVE_MARKERS = [ + "input/raw", + "workspace/raw", + "workspace/chunks", + "workspace/clean_notes", + "workspace/source_cards", + ".temp", + ".tmp", + ".log", + ".env", + "embeddings", + "vector db", + "vector_db", + "qdrant", + "sqlite vector", + "semantic_rag", +] + + +@dataclass(frozen=True) +class GateResult: + gate_name: str + quality_status: str + reason: str + unsupported_claims: list[str] + required_revision: list[str] + residual_risks: list[str] + + +@dataclass(frozen=True) +class ReleaseGateReport: + package_dir: str + governance_dir: str + scope: str + mode: str + quality_status: str + gates: list[GateResult] + failed_gates: list[str] + blocked_items: list[str] + unsupported_claims: list[str] + required_revision: list[str] + residual_risks: list[str] + + +def evaluate_release( + package_dir: Path | None, + governance_dir: Path | None, + scope: str = "full", + mode: str = "candidate", +) -> ReleaseGateReport: + if scope not in SCOPES: + raise ValueError(f"scope must be one of: {', '.join(sorted(SCOPES))}") + if mode not in MODES: + raise ValueError(f"mode must be one of: {', '.join(sorted(MODES))}") + + package_path = Path(package_dir) if package_dir is not None else None + governance_path = Path(governance_dir) if governance_dir is not None else None + gates = [ + _gate_governance_exists(governance_path, scope), + _gate_governance_required_files(governance_path, scope), + _gate_package_exists(package_path, scope), + _gate_package_required_files(package_path, scope), + _gate_boundary_scan(package_path, governance_path, scope), + _gate_card_passport_schema(package_path, governance_path, scope), + _gate_confidence_review_status(package_path, scope), + _gate_weak_unsupported_promotion(package_path, scope), + _gate_negative_eval_presence(governance_path, scope), + _gate_acceptance_decision(package_path, mode, scope), + _gate_promotion_readiness(package_path, mode, scope), + ] + quality_status = _overall_status(gates) + failed_gates = [gate.gate_name for gate in gates if gate.quality_status != "pass"] + blocked_items = [gate.reason for gate in gates if gate.quality_status == "blocked"] + unsupported_claims = _flatten(gate.unsupported_claims for gate in gates) + required_revision = _flatten(gate.required_revision for gate in gates) + residual_risks = _flatten(gate.residual_risks for gate in gates) + + return ReleaseGateReport( + package_dir=package_path.as_posix() if package_path is not None else "", + governance_dir=governance_path.as_posix() if governance_path is not None else "", + scope=scope, + mode=mode, + quality_status=quality_status, + gates=gates, + failed_gates=failed_gates, + blocked_items=blocked_items, + unsupported_claims=unsupported_claims, + required_revision=required_revision, + residual_risks=residual_risks, + ) + + +def report_to_dict(report: ReleaseGateReport) -> dict: + return asdict(report) + + +def report_to_markdown(report: ReleaseGateReport) -> str: + lines = [ + "# KB Release Gate Report", + "", + f"- package_dir: {report.package_dir or '(not provided)'}", + f"- governance_dir: {report.governance_dir or '(not provided)'}", + f"- scope: {report.scope}", + f"- mode: {report.mode}", + f"- quality_status: {report.quality_status}", + "", + "## Gates", + "", + "| Gate | quality_status | reason | unsupported_claims | required_revision | residual_risks |", + "|---|---|---|---|---|---|", + ] + for gate in report.gates: + lines.append( + "| " + + " | ".join( + [ + _escape_table(gate.gate_name), + _escape_table(gate.quality_status), + _escape_table(gate.reason), + _escape_table("; ".join(gate.unsupported_claims) or "-"), + _escape_table("; ".join(gate.required_revision) or "-"), + _escape_table("; ".join(gate.residual_risks) or "-"), + ] + ) + + " |" + ) + return "\n".join(lines) + "\n" + + +def _gate_governance_exists(governance_dir: Path | None, scope: str) -> GateResult: + if scope == "package": + return _pass("governance_exists", "governance not required for package-only scope") + if governance_dir is None: + return _blocked("governance_exists", "governance directory not provided", ["Provide --governance-dir."]) + if not governance_dir.exists() or not governance_dir.is_dir(): + return _blocked("governance_exists", f"governance directory missing: {governance_dir}", ["Create docs/kb_release."]) + return _pass("governance_exists", "governance directory exists") + + +def _gate_governance_required_files(governance_dir: Path | None, scope: str) -> GateResult: + if scope == "package": + return _pass("governance_required_files", "governance not required for package-only scope") + if governance_dir is None or not governance_dir.is_dir(): + return _blocked("governance_required_files", "governance required files cannot be checked", ["Provide governance docs."]) + missing = [name for name in REQUIRED_GOVERNANCE_FILES if not (governance_dir / name).is_file()] + if missing: + return _revise("governance_required_files", "governance files missing", [f"Add: {', '.join(missing)}"]) + return _pass("governance_required_files", "all required governance files exist") + + +def _gate_package_exists(package_dir: Path | None, scope: str) -> GateResult: + if scope == "governance-only": + return _pass("package_exists", "package not required for governance-only scope") + if package_dir is None: + return _blocked("package_exists", "package directory not provided", ["Provide --package-dir."]) + if not package_dir.exists() or not package_dir.is_dir(): + return _blocked("package_exists", f"package directory missing: {package_dir}", ["Build or provide a candidate package."]) + return _pass("package_exists", "package directory exists") + + +def _gate_package_required_files(package_dir: Path | None, scope: str) -> GateResult: + if scope == "governance-only": + return _pass("package_required_files", "package not required for governance-only scope") + if package_dir is None or not package_dir.is_dir(): + return _blocked("package_required_files", "package required files cannot be checked", ["Provide a candidate package."]) + missing = [name for name in REQUIRED_COMPACT_FILES if not (package_dir / name).is_file()] + if missing: + return _revise("package_required_files", "required compact package files missing", [f"Add: {', '.join(missing)}"]) + return _pass("package_required_files", "all required compact package files exist") + + +def _gate_boundary_scan(package_dir: Path | None, governance_dir: Path | None, scope: str) -> GateResult: + roots = [] + if scope in {"governance-only", "full"} and governance_dir is not None and governance_dir.is_dir(): + roots.append(governance_dir) + if scope in {"package", "full"} and package_dir is not None and package_dir.is_dir(): + roots.append(package_dir) + findings = [] + for root in roots: + for path in _iter_text_files(root): + for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + if _is_active_forbidden_line(line): + findings.append(f"{path.relative_to(root)}:{line_number}") + if findings: + return _blocked( + "boundary_scan", + "active source or implementation claim references forbidden material", + ["Remove active forbidden source/include claims."], + unsupported_claims=findings, + ) + return _pass("boundary_scan", "no active forbidden source markers found") + + +def _gate_card_passport_schema(package_dir: Path | None, governance_dir: Path | None, scope: str) -> GateResult: + paths = [] + if scope in {"governance-only", "full"} and governance_dir is not None: + paths.append(governance_dir / "KB_CARD_PASSPORT.md") + if scope in {"package", "full"} and package_dir is not None: + paths.append(package_dir / "KB__CARD_SCHEMA.md") + existing = [path for path in paths if path.is_file()] + if not existing: + return _revise("card_passport_schema", "card passport schema file missing", ["Add card passport/schema documentation."]) + combined = "\n".join(path.read_text(encoding="utf-8").lower() for path in existing) + missing = [field for field in CARD_PASSPORT_FIELDS if field not in combined] + if missing: + return _revise("card_passport_schema", "card passport fields missing", [f"Document fields: {', '.join(missing)}"]) + return _pass("card_passport_schema", "card passport schema includes required fields") + + +def _gate_confidence_review_status(package_dir: Path | None, scope: str) -> GateResult: + if scope == "governance-only": + return _pass("confidence_review_status", "package status values not required for governance-only scope") + if package_dir is None or not package_dir.is_dir(): + return _blocked("confidence_review_status", "package status values cannot be checked", ["Provide a candidate package."]) + text = _read_optional(package_dir / "KB__CONFIDENCE_RULES.md").lower() + missing_confidence = [value for value in sorted(ALLOWED_CONFIDENCE) if value not in text] + missing_review = [value for value in sorted(ALLOWED_REVIEW_STATUS) if value not in text] + if missing_confidence or missing_review: + revision = [] + if missing_confidence: + revision.append(f"Document confidence values: {', '.join(missing_confidence)}") + if missing_review: + revision.append(f"Document review_status values: {', '.join(missing_review)}") + return _revise("confidence_review_status", "confidence or review status values missing", revision) + return _pass("confidence_review_status", "confidence and review status values are documented") + + +def _gate_weak_unsupported_promotion(package_dir: Path | None, scope: str) -> GateResult: + if scope == "governance-only": + return _pass("weak_unsupported_promotion", "package promotion claims not required for governance-only scope") + if package_dir is None or not package_dir.is_dir(): + return _blocked("weak_unsupported_promotion", "package promotion claims cannot be checked", ["Provide a candidate package."]) + findings = [] + pattern = re.compile(r"(weak|unsupported).{0,80}(promoted as fact|promote[d]? to strong|accepted as supported)", re.I) + for path in _iter_text_files(package_dir): + for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + if pattern.search(line): + findings.append(f"{path.relative_to(package_dir)}:{line_number}") + if findings: + return _blocked( + "weak_unsupported_promotion", + "weak or unsupported evidence is promoted beyond its status", + ["Keep weak/unsupported evidence out of strong facts."], + unsupported_claims=findings, + ) + return _pass("weak_unsupported_promotion", "weak and unsupported promotion blockers not found") + + +def _gate_negative_eval_presence(governance_dir: Path | None, scope: str) -> GateResult: + if scope == "package": + return _pass("negative_eval_presence", "governance negative eval cases not required for package-only scope") + path = governance_dir / "NEGATIVE_EVAL_CASES.md" if governance_dir is not None else None + if path is None or not path.is_file(): + return _revise("negative_eval_presence", "negative eval cases file missing", ["Add NEGATIVE_EVAL_CASES.md."]) + text = path.read_text(encoding="utf-8") + missing = [case_id for case_id in sorted(NEGATIVE_EVAL_CASE_IDS) if case_id not in text] + if missing: + return _revise("negative_eval_presence", "negative eval cases missing", [f"Add cases: {', '.join(missing)}"]) + return _pass("negative_eval_presence", "required negative eval cases are present") + + +def _gate_acceptance_decision(package_dir: Path | None, mode: str, scope: str) -> GateResult: + if mode == "candidate": + return _pass("acceptance_decision", "acceptance decision is not required for candidate mode") + if scope == "governance-only": + return _blocked("acceptance_decision", "promotion mode requires a package acceptance decision", ["Run promotion mode with --package-dir."]) + if package_dir is None or not package_dir.is_dir(): + return _blocked("acceptance_decision", "acceptance decision cannot be checked", ["Provide a candidate package."]) + decision = _find_acceptance(package_dir) + if decision is None: + return _blocked("acceptance_decision", "accepted YAML acceptance block is absent", ["Add accepted acceptance block."]) + if not _is_accepted(decision): + return _blocked("acceptance_decision", "acceptance decision is not accepted for strong promotion", ["Set accepted decision only after all gates pass."]) + return _pass("acceptance_decision", "accepted YAML acceptance block is present") + + +def _gate_promotion_readiness(package_dir: Path | None, mode: str, scope: str) -> GateResult: + if mode == "candidate": + return _pass("promotion_readiness", "promotion readiness is not required for candidate mode") + if scope == "governance-only": + return _blocked("promotion_readiness", "promotion readiness requires a package", ["Run promotion mode with --package-dir."]) + if package_dir is None or not package_dir.is_dir(): + return _blocked("promotion_readiness", "promotion readiness cannot be checked", ["Provide a candidate package."]) + decision = _find_acceptance(package_dir) + if decision is None or not _is_accepted(decision): + return _blocked("promotion_readiness", "package is not ready for strong promotion", ["Complete acceptance before promotion."]) + return _pass("promotion_readiness", "package is ready for strong promotion after acceptance") + + +def _find_acceptance(package_dir: Path) -> dict[str, str] | None: + for path in _iter_text_files(package_dir): + parsed = _parse_acceptance_block(path.read_text(encoding="utf-8")) + if parsed is not None: + return parsed + return None + + +def _parse_acceptance_block(text: str) -> dict[str, str] | None: + lines = text.splitlines() + for index, line in enumerate(lines): + if line.strip() != "acceptance:": + continue + values: dict[str, str] = {} + for child in lines[index + 1 : index + 5]: + if not child.startswith(" "): + break + key, separator, value = child.strip().partition(":") + if separator: + values[key] = value.strip() + if {"quality_status", "decision", "promote_to_strong"} <= set(values): + return values + return None + + +def _is_accepted(decision: dict[str, str]) -> bool: + return ( + decision.get("quality_status") == "pass" + and decision.get("decision") == "accepted" + and decision.get("promote_to_strong") == "yes" + ) + + +def _is_active_forbidden_line(line: str) -> bool: + lower = line.strip().lower() + if not any(marker in lower for marker in FORBIDDEN_ACTIVE_MARKERS): + return False + if lower.startswith("\n", + encoding="utf-8", + ) + + report = evaluate_release(package, governance, scope="full", mode="candidate") + + assert report.quality_status == "blocked" + + +def test_env_reference_as_source_returns_blocked(tmp_path): + governance = write_governance(tmp_path) + package = write_package(tmp_path) + package.joinpath("MANIFEST.md").write_text("source: .env\n", encoding="utf-8") + + report = evaluate_release(package, governance, scope="full", mode="candidate") + + assert report.quality_status == "blocked" + + +def test_policy_mention_of_raw_is_not_blocked(tmp_path): + governance = write_governance(tmp_path) + package = write_package(tmp_path) + package.joinpath("README.md").write_text("Do not include input/raw.\n", encoding="utf-8") + + report = evaluate_release(package, governance, scope="full", mode="candidate") + + assert report.quality_status == "pass" + + +def test_policy_mention_of_embeddings_is_not_blocked(tmp_path): + governance = write_governance(tmp_path) + package = write_package(tmp_path) + package.joinpath("README.md").write_text("This package does not use embeddings.\n", encoding="utf-8") + + report = evaluate_release(package, governance, scope="full", mode="candidate") + + assert report.quality_status == "pass" + + +def test_promotion_without_acceptance_is_blocked(tmp_path): + governance = write_governance(tmp_path) + package = write_package(tmp_path) + + report = evaluate_release(package, governance, scope="full", mode="promotion") + + assert report.quality_status == "blocked" + assert "acceptance_decision" in report.failed_gates + + +def test_promotion_with_accepted_decision_passes(tmp_path): + governance = write_governance(tmp_path) + package = write_package(tmp_path, accepted=True) + + report = evaluate_release(package, governance, scope="full", mode="promotion") + + assert report.quality_status == "pass" + + +def test_promotion_with_promote_no_is_blocked(tmp_path): + governance = write_governance(tmp_path) + package = write_package(tmp_path, accepted=True, promote_to_strong="no") + + report = evaluate_release(package, governance, scope="full", mode="promotion") + + assert report.quality_status == "blocked" + + +def test_overall_status_precedence_blocked_over_revise(tmp_path): + governance = write_governance(tmp_path) + package = write_package(tmp_path) + package.joinpath("KB__00_INDEX.md").unlink() + package.joinpath("MANIFEST.md").write_text("source: .env\n", encoding="utf-8") + + report = evaluate_release(package, governance, scope="full", mode="candidate") + + assert report.quality_status == "blocked" + assert {"package_required_files", "boundary_scan"} <= set(report.failed_gates) + + +def test_json_report_shape(tmp_path): + governance = write_governance(tmp_path) + + payload = report_to_dict(evaluate_release(None, governance, scope="governance-only", mode="candidate")) + + assert {"package_dir", "governance_dir", "scope", "mode", "quality_status", "gates"} <= set(payload) + assert {"gate_name", "quality_status", "reason"} <= set(payload["gates"][0]) + + +def test_markdown_report_contains_gate_table(tmp_path): + governance = write_governance(tmp_path) + + markdown = report_to_markdown(evaluate_release(None, governance, scope="governance-only", mode="candidate")) + + assert "| Gate | quality_status | reason |" in markdown + + +def test_cli_json_exit_code_pass(tmp_path): + governance = write_governance(tmp_path) + + result = subprocess.run( + [ + "python3", + str(SCRIPT), + "--governance-dir", + str(governance), + "--scope", + "governance-only", + "--mode", + "candidate", + "--json", + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout)["quality_status"] == "pass" + + +def test_cli_exit_code_blocked(tmp_path): + result = subprocess.run( + ["python3", str(SCRIPT), "--package-dir", str(tmp_path / "missing"), "--scope", "package"], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 2 + + +def test_cli_governance_only_without_package(tmp_path): + governance = write_governance(tmp_path) + + result = subprocess.run( + [ + "python3", + str(SCRIPT), + "--governance-dir", + str(governance), + "--scope", + "governance-only", + "--json", + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0 + + +def test_no_network_or_llm_dependency(tmp_path): + governance = write_governance(tmp_path) + + report = evaluate_release(None, governance, scope="governance-only", mode="candidate") + + assert report.quality_status == "pass" + assert "requests" not in SCRIPT.read_text(encoding="utf-8") + assert "urllib" not in SCRIPT.read_text(encoding="utf-8") diff --git a/tests/test_search_cli_mvp_10.py b/tests/test_search_cli_mvp_10.py index 4b0ffad..9238b9f 100644 --- a/tests/test_search_cli_mvp_10.py +++ b/tests/test_search_cli_mvp_10.py @@ -9,7 +9,15 @@ ROOT = Path(__file__).resolve().parents[1] SCRIPT = ROOT / "scripts" / "search_kb.py" -FULL_KB = ROOT / "publish" / "markdown_kb" / "full_kb.md" + + +def write_kb_file(tmp_path: Path) -> Path: + kb_file = tmp_path / "full_kb.md" + kb_file.write_text( + "Alpha\nKnowledge Base pipeline\nKnowledge card\nOmega\n", + encoding="utf-8", + ) + return kb_file def test_search_text_finds_case_insensitive_snippet(): @@ -36,9 +44,11 @@ def test_search_file_rejects_missing_file(tmp_path): search_file(tmp_path / "missing.md", query="KB") -def test_cli_text_output_returns_known_query(): +def test_cli_text_output_returns_known_query(tmp_path): + kb_file = write_kb_file(tmp_path) + result = subprocess.run( - ["python3", str(SCRIPT), "Knowledge", "--limit", "2"], + ["python3", str(SCRIPT), "Knowledge", "--path", str(kb_file), "--limit", "2"], cwd=ROOT, check=False, capture_output=True, @@ -51,9 +61,11 @@ def test_cli_text_output_returns_known_query(): assert "Knowledge" in result.stdout -def test_cli_json_output_is_valid_and_limited(): +def test_cli_json_output_is_valid_and_limited(tmp_path): + kb_file = write_kb_file(tmp_path) + result = subprocess.run( - ["python3", str(SCRIPT), "Knowledge", "--limit", "1", "--json"], + ["python3", str(SCRIPT), "Knowledge", "--path", str(kb_file), "--limit", "1", "--json"], cwd=ROOT, check=False, capture_output=True, @@ -63,7 +75,7 @@ def test_cli_json_output_is_valid_and_limited(): assert result.returncode == 0, result.stderr payload = json.loads(result.stdout) assert payload["query"] == "Knowledge" - assert payload["path"] == FULL_KB.as_posix() + assert payload["path"] == kb_file.as_posix() assert len(payload["results"]) <= 1 assert {"line_number", "snippet"} <= set(payload["results"][0]) @@ -81,9 +93,11 @@ def test_cli_rejects_missing_file(tmp_path): assert "search file not found" in result.stderr -def test_cli_rejects_empty_query(): +def test_cli_rejects_empty_query(tmp_path): + kb_file = write_kb_file(tmp_path) + result = subprocess.run( - ["python3", str(SCRIPT), " ", "--path", str(FULL_KB)], + ["python3", str(SCRIPT), " ", "--path", str(kb_file)], cwd=ROOT, check=False, capture_output=True,