From f4faed9f80042b778d0a355f8c9a30b81a1f1b35 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:40:04 +0800 Subject: [PATCH] Fix current compatibility checks for constraint metadata and optional AI Co-Authored-By: Codex --- .github/workflows/qsl_exception_lifecycle.yml | 13 +- docs/qsl_compat_upgrade.md | 2 +- docs/qsl_version_control.md | 4 +- python/tests/test_qsl_compat_checker.py | 114 +++++++++++++++-- .../test_qsl_exception_lifecycle_workflow.py | 53 ++++++++ scripts/check_qsl_compat.py | 119 ++++++++++++++---- 6 files changed, 263 insertions(+), 42 deletions(-) diff --git a/.github/workflows/qsl_exception_lifecycle.yml b/.github/workflows/qsl_exception_lifecycle.yml index 75a4cf6..167dfdf 100644 --- a/.github/workflows/qsl_exception_lifecycle.yml +++ b/.github/workflows/qsl_exception_lifecycle.yml @@ -204,9 +204,12 @@ jobs: set -euo pipefail marker="${RUNNER_TEMP}/qsl-aiauditbridge-identity-created" if [ -f "$marker" ]; then - destination="$(cat "$marker")" - case "$destination" in - "${GITHUB_WORKSPACE}/../AIAuditBridge") rm -rf "$destination" "$marker" ;; - *) echo "refusing to clean unexpected dependency identity path: $destination" >&2; exit 1 ;; - esac + destination="$(realpath -m "$(cat "$marker")")" + expected="$(realpath -m "${GITHUB_WORKSPACE}/../AIAuditBridge")" + if [ "$destination" = "$expected" ]; then + rm -rf "$destination" "$marker" + else + echo "refusing to clean unexpected dependency identity path: $destination" >&2 + exit 1 + fi fi diff --git a/docs/qsl_compat_upgrade.md b/docs/qsl_compat_upgrade.md index 8039f58..eca997b 100644 --- a/docs/qsl_compat_upgrade.md +++ b/docs/qsl_compat_upgrade.md @@ -17,7 +17,7 @@ This repository defines the QSL central compatibility manifest and its upgrade p - 禁止短 SHA - 禁止 `requirements.txt` / `constraints.txt`（未设置 `allow_legacy=true` 时） - 内部依赖 Ref 是否与 `compat/bundles/.toml` 一致 - - `--scope current` 按 consumer 自己声明的 `qsl.requires`，核对 `pyproject.toml`、`uv.lock` 和适用 legacy 文件中的受管 git refs；仍拒绝缺声明、短 SHA、`main`、manifest/lock 冲突、unmanaged source 和禁止依赖方向，但不把 bundle 差异当作 current 一致性结论。 + - `--scope current` 按 consumer 自己声明的 `qsl.requires`，核对 `pyproject.toml`、`uv.lock` 和适用 legacy 文件中的受管 git refs；标记为 `live_constraint_files` 的 legacy 文件只有在与同仓 `qsl-pins.txt` 逐条 exact mirror（来源与完整 SHA 均正确）时才作为 pin metadata，不作为安装依赖。镜像缺失、格式错误或不一致仍失败；其他 legacy 文件仍按实际依赖检查。仍拒绝缺声明、短 SHA、`main`、manifest/lock 冲突、unmanaged source 和禁止依赖方向，但不把 bundle 差异当作 current 一致性结论。 - `scripts/render_qsl_dependency_graph.py` - 输出当前仓库的 QSL 依赖图（Markdown / Text）。 diff --git a/docs/qsl_version_control.md b/docs/qsl_version_control.md index 0eaedc6..9ec7796 100644 --- a/docs/qsl_version_control.md +++ b/docs/qsl_version_control.md @@ -7,7 +7,7 @@ management. - `compat/bundles/*.toml` is the source of truth for internal repository commit pins. - Each consumer repository declares its bundle in `qsl.toml`. -- Consumer files (`pyproject.toml`, `uv.lock`, `requirements.txt`, `constraints.txt`) must match the declared bundle. +- Consumer installation files (`pyproject.toml`, `uv.lock`, and actual legacy requirements) must match the declared bundle. A `live_constraint_files` entry such as `constraints.txt` is treated as pin metadata only when it is an exact source/full-SHA mirror of the same repository's `qsl-pins.txt`; malformed or mismatched mirrors fail validation. - `internal_dependency_matrix.json` is generated from local consumer dependency files; do not hand-edit it except for emergency repair. ## CLI @@ -36,7 +36,7 @@ python3 python/scripts/qslctl.py generate-matrix --projects-root /Users/lisiyi/P - `qslctl report` is read-only. It groups the current workspace by ring, status, and bundle hotspot. - `qslctl plan` is read-only. It renders the ring-by-ring convergence order and highlights which repos should be fixed before the next ring starts. -- Both commands default to `--scope frozen`, preserving exact bundle ref checks. `--scope current` checks each consumer's `qsl.requires` against its manifests, locks, and applicable legacy constraints; `CONSISTENT` then means current declaration consistency only. Different valid full SHAs across consumers do not alone require `HUMAN_REQUIRED`, while actual issues, missing repositories, and invalid configuration still do. +- Both commands default to `--scope frozen`, preserving exact bundle ref checks. `--scope current` checks each consumer's `qsl.requires` against its manifests, locks, and actual legacy dependencies; a `live_constraint_files` mirror is checked against `qsl-pins.txt` for metadata integrity without becoming an install dependency. `CONSISTENT` then means current declaration consistency only. Different valid full SHAs across consumers do not alone require `HUMAN_REQUIRED`, while actual issues, missing repositories, and invalid configuration still do. - Use `report` to answer “what is broken right now?” and `plan` to answer “what should we fix first?” ## QSL exception lifecycle check diff --git a/python/tests/test_qsl_compat_checker.py b/python/tests/test_qsl_compat_checker.py index 5a24158..b804b35 100644 --- a/python/tests/test_qsl_compat_checker.py +++ b/python/tests/test_qsl_compat_checker.py @@ -96,7 +96,11 @@ def test_current_scope_accepts_declared_manifest_ref_even_when_bundle_differs(se with tempfile.TemporaryDirectory() as workspace: compat_root = Path(workspace) self._write_repo_tiers(compat_root) - self._write_bundle(compat_root, "2026.07.2", {"QuantPlatformKit": "a" * 40}) + self._write_bundle( + compat_root, + "2026.07.2", + {"QuantPlatformKit": "a" * 40, "UsEquityStrategies": "b" * 40}, + ) repo_root = self._make_repo_root( qsl_toml=( 'tier = "ops/tooling"\n' @@ -169,7 +173,11 @@ def test_current_rejects_missing_direct_dependency_from_lock(self): with tempfile.TemporaryDirectory() as workspace: compat_root = Path(workspace) self._write_repo_tiers(compat_root) - self._write_bundle(compat_root, "2026.07.2", {"QuantPlatformKit": "a" * 40}) + self._write_bundle( + compat_root, + "2026.07.2", + {"QuantPlatformKit": "a" * 40, "UsEquityStrategies": "b" * 40}, + ) repo_root = self._make_repo_root( qsl_toml=( 'tier = "ops/tooling"\nupgrade_ring = "ring_e"\n[compat]\n' @@ -188,8 +196,93 @@ def test_current_rejects_missing_direct_dependency_from_lock(self): self.assertFalse(ok) self.assertTrue(any("current lock missing direct dependency" in issue for issue in issues)) - def test_current_allows_research_only_ops_dependency_but_base_dependency_is_blocked(self): - def run(base_dependency: bool, marker: str = "extra == 'research'"): + def test_current_treats_exact_qpk_constraint_mirror_as_pin_metadata(self): + with tempfile.TemporaryDirectory() as workspace: + compat_root = Path(workspace) + self._write_repo_tiers(compat_root) + self._write_bundle( + compat_root, + "2026.07.2", + {"QuantPlatformKit": "a" * 40, "UsEquityStrategies": "b" * 40}, + ) + repo_root = self._make_repo_root( + qsl_toml=( + 'tier = "core"\nupgrade_ring = "ring_a"\nallow_legacy = true\n' + 'legacy_reason = "qsl-pins mirror"\n[compat]\nbundle = "2026.07.2"\n' + 'live_constraint_files = ["constraints.txt"]\n' + ), + pyproject="", + ) + repo_root = repo_root.rename(Path(workspace) / "QuantPlatformKit") + entries = ( + "quant-platform-kit @ git+https://github.com/QuantStrategyLab/QuantPlatformKit.git@" + "a" * 40, + "us-equity-strategies @ git+https://github.com/QuantStrategyLab/UsEquityStrategies.git@" + "b" * 40, + ) + (repo_root / "qsl-pins.txt").write_text("# source\n" + "\n".join(entries) + "\n", encoding="utf-8") + (repo_root / "constraints.txt").write_text("# mirror\n" + "\n".join(entries) + "\n", encoding="utf-8") + ok, issues, _warnings, _notes = check_qsl_compat._check( + repo_root=repo_root, compat_root=compat_root, scope="current" + ) + self.assertTrue(ok, issues) + + strategy_root = Path(workspace) / "CnEquityStrategies" + strategy_root.mkdir() + for name in ("qsl.toml", "pyproject.toml", "qsl-pins.txt", "constraints.txt"): + (strategy_root / name).write_text((repo_root / name).read_text(encoding="utf-8"), encoding="utf-8") + blocked, strategy_issues, _warnings, _notes = check_qsl_compat._check( + repo_root=strategy_root, compat_root=compat_root, scope="current" + ) + self.assertFalse(blocked) + self.assertTrue(any("missing current qsl.requires declaration" in issue for issue in strategy_issues)) + + def test_current_rejects_invalid_or_mismatched_qpk_constraint_mirror(self): + with tempfile.TemporaryDirectory() as workspace: + compat_root = Path(workspace) + self._write_repo_tiers(compat_root) + self._write_bundle(compat_root, "2026.07.2", {"QuantPlatformKit": "a" * 40}) + repo_root = self._make_repo_root( + qsl_toml=( + 'tier = "core"\nupgrade_ring = "ring_a"\nallow_legacy = true\n' + '[compat]\nbundle = "2026.07.2"\nlive_constraint_files = ["constraints.txt"]\n' + ), + pyproject="", + ) + repo_root = repo_root.rename(Path(workspace) / "QuantPlatformKit") + (repo_root / "qsl-pins.txt").write_text( + "quant-platform-kit @ git+https://github.com/QuantStrategyLab/QuantPlatformKit.git@" + "a" * 40 + "\n", + encoding="utf-8", + ) + (repo_root / "constraints.txt").write_text( + "quant-platform-kit @ https://example.com/QuantPlatformKit.git@main\n", encoding="utf-8" + ) + ok, issues, _warnings, _notes = check_qsl_compat._check( + repo_root=repo_root, compat_root=compat_root, scope="current" + ) + self.assertFalse(ok) + self.assertTrue(any("invalid qsl pin mirror entry" in issue for issue in issues)) + self.assertTrue(any("qsl pin mirror mismatch" in issue for issue in issues)) + + (repo_root / "qsl-pins.txt").write_text( + "unknown @ git+https://github.com/QuantStrategyLab/UnknownRepo.git@" + "c" * 40 + "\n", + encoding="utf-8", + ) + (repo_root / "constraints.txt").write_text( + "unknown @ git+https://github.com/QuantStrategyLab/UnknownRepo.git@" + "c" * 40 + "\n", + encoding="utf-8", + ) + ok, issues, _warnings, _notes = check_qsl_compat._check( + repo_root=repo_root, compat_root=compat_root, scope="current" + ) + self.assertFalse(ok) + self.assertTrue(any("unmanaged qsl pin mirror entry" in issue for issue in issues)) + + def test_current_allows_approved_optional_ops_dependency_but_base_dependency_is_blocked(self): + def run( + base_dependency: bool, + marker: str = "extra == 'research'", + extra_name: str = "research", + duplicate_marker: str | None = None, + ): with tempfile.TemporaryDirectory() as workspace: root = Path(workspace) compat_root = root / "QuantRuntimeSettings" @@ -213,13 +306,15 @@ def run(base_dependency: bool, marker: str = "extra == 'research'"): deps = (base + ',') if base else '' (repo_root / "pyproject.toml").write_text( '[project]\nname = "consumer"\ndependencies = [' + deps + ']\n' - '[project.optional-dependencies]\nresearch = ["ai-gateway-client @ git+https://github.com/QuantStrategyLab/' + '[project.optional-dependencies]\n' + extra_name + ' = ["ai-gateway-client @ git+https://github.com/QuantStrategyLab/' + "AIAuditBridge.git@" + ref + '"]\n', encoding="utf-8" ) + lock_requires = '{ name = "ai-gateway-client", marker = "' + marker + '", git = "https://github.com/QuantStrategyLab/AIAuditBridge.git?rev=' + ref + '" }' + if duplicate_marker is not None: + lock_requires += ', { name = "ai-gateway-client", marker = "' + duplicate_marker + '", git = "https://github.com/QuantStrategyLab/AIAuditBridge.git?rev=' + ref + '" }' (repo_root / "uv.lock").write_text( 'version = 1\n[[package]]\nname = "consumer"\nsource = { editable = "." }\n' - '[package.metadata]\nrequires-dist = [{ name = "ai-gateway-client", marker = "' + marker + '", git = "https://github.com/QuantStrategyLab/AIAuditBridge.git?rev=' - + ref + '" }]\n\n[[package]]\nname = "ai-gateway-client"\nsource = { git = "https://github.com/QuantStrategyLab/AIAuditBridge.git?rev=' + '[package.metadata]\nrequires-dist = [' + lock_requires + ']\n\n[[package]]\nname = "ai-gateway-client"\nsource = { git = "https://github.com/QuantStrategyLab/AIAuditBridge.git?rev=' + ref + '" }\n', encoding="utf-8" ) return check_qsl_compat._check(repo_root=repo_root, compat_root=compat_root, scope="current") @@ -232,6 +327,11 @@ def run(base_dependency: bool, marker: str = "extra == 'research'"): complex_marker = run(False, "extra == 'research' or extra == 'test'") self.assertFalse(complex_marker[0]) self.assertTrue(any("forbidden dependency direction" in issue for issue in complex_marker[1])) + ai_extra = run(False, "extra == 'ai'", "ai") + self.assertTrue(ai_extra[0], ai_extra[1]) + duplicate_marker = run(False, duplicate_marker="extra == 'research' or extra == 'test'") + self.assertFalse(duplicate_marker[0]) + self.assertTrue(any("forbidden dependency direction" in issue for issue in duplicate_marker[1])) def test_frozen_scope_keeps_historical_2026090_b13_mismatch_blocked(self): with tempfile.TemporaryDirectory() as workspace: diff --git a/python/tests/test_qsl_exception_lifecycle_workflow.py b/python/tests/test_qsl_exception_lifecycle_workflow.py index 8ab1494..7864e79 100644 --- a/python/tests/test_qsl_exception_lifecycle_workflow.py +++ b/python/tests/test_qsl_exception_lifecycle_workflow.py @@ -3,7 +3,11 @@ import unittest import tempfile import importlib.util +import os +import re +import subprocess import sys +import textwrap from pathlib import Path @@ -32,6 +36,55 @@ def test_workflow_keeps_frozen_checkout_and_current_main_sha_recording(self) -> self.assertIn('git -C "$workspace/$repo" checkout --detach origin/main', workflow) self.assertIn('git -C "$workspace/$repo" rev-parse HEAD', workflow) self.assertIn('if report["strict_repositories"] or report["warning_repositories"]:', workflow) + self.assertIn('destination="$(realpath -m "$(cat "$marker")")"', workflow) + self.assertIn('expected="$(realpath -m "${GITHUB_WORKSPACE}/../AIAuditBridge")"', workflow) + + def test_cleanup_shell_normalizes_resolved_identity_marker_path(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + workspace = root / "runner" / "work" / "QRT" + identity = root / "runner" / "work" / "AIAuditBridge" + workspace.mkdir(parents=True) + identity.mkdir() + workflow = (ROOT / ".github/workflows/qsl_exception_lifecycle.yml").read_text(encoding="utf-8") + match = re.search( + r"- name: Clean up research dependency identity\n" + r"\s+if: always\(\)\n\s+run: \|\n((?:\s{10}.*\n)+)", + workflow, + ) + self.assertIsNotNone(match) + script = textwrap.dedent(match.group(1)) + # macOS ships a realpath without GNU's -m; provide the same tiny + # command surface locally while exercising the workflow shell. + bin_dir = root / "bin" + bin_dir.mkdir() + (bin_dir / "realpath").write_text( + "#!/usr/bin/env python3\n" + "import os, sys\n" + "args = sys.argv[1:]\n" + "if args and args[0] == '-m': args = args[1:]\n" + "print(os.path.abspath(os.path.normpath(args[0])))\n", + encoding="utf-8", + ) + (bin_dir / "realpath").chmod(0o755) + env = { + "RUNNER_TEMP": str(root / "runner"), + "GITHUB_WORKSPACE": str(workspace.resolve()), + "PATH": f"{bin_dir}:{os.environ.get('PATH', '')}", + } + marker = root / "runner" / "qsl-aiauditbridge-identity-created" + marker.write_text(str(identity.resolve()), encoding="utf-8") + subprocess.run(["bash", "-c", script], check=True, env=env, capture_output=True, text=True) + self.assertFalse(identity.exists()) + self.assertFalse(marker.exists()) + + unexpected = root / "runner" / "unexpected-AIAuditBridge" + unexpected.mkdir() + marker.write_text(str(unexpected.resolve()), encoding="utf-8") + result = subprocess.run(["bash", "-c", script], check=False, env=env, capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertTrue(unexpected.exists()) + self.assertTrue(marker.exists()) def test_workflow_layout_current_checks_four_consumers_and_keeps_direction_fail_closed(self) -> None: with tempfile.TemporaryDirectory() as directory: diff --git a/scripts/check_qsl_compat.py b/scripts/check_qsl_compat.py index 6d8596d..f385a46 100755 --- a/scripts/check_qsl_compat.py +++ b/scripts/check_qsl_compat.py @@ -300,7 +300,7 @@ def _validate_dependency_direction( def _is_research_extra_only_dependency(repo_root: Path, pin: GitRef) -> bool: - """Prove a git dependency is reachable only through the research extra. + """Prove a git dependency is reachable only through an approved AI extra. The manifest and uv lock must independently identify the dependency as a research-only requirement. This deliberately does not treat arbitrary @@ -319,25 +319,38 @@ def _is_research_extra_only_dependency(repo_root: Path, pin: GitRef) -> bool: if not isinstance(project, dict): return False optional = project.get("optional-dependencies") - research = optional.get("research") if isinstance(optional, dict) else None - if not isinstance(research, list): - return False - dependency_name = next( - ( - str(item).split("@", 1)[0].strip().lower() - for item in research - if isinstance(item, str) and "QuantStrategyLab/" + pin.repo + ".git@" in item - ), - None, - ) - if not dependency_name: + if not isinstance(optional, dict): return False base_dependencies = project.get("dependencies", []) - if isinstance(base_dependencies, list) and any( - isinstance(item, str) and item.split("@", 1)[0].strip().lower() == dependency_name - for item in base_dependencies - ): - return False + for extra_name in ("ai", "research"): + requirements = optional.get(extra_name) + if not isinstance(requirements, list): + continue + dependency_name = next( + ( + str(item).split("@", 1)[0].strip().lower() + for item in requirements + if isinstance(item, str) and "QuantStrategyLab/" + pin.repo + ".git@" in item + ), + None, + ) + if not dependency_name: + continue + if isinstance(base_dependencies, list) and any( + isinstance(item, str) and item.split("@", 1)[0].strip().lower() == dependency_name + for item in base_dependencies + ): + continue + if _lock_proves_optional_git_dependency( + lock=lock, dependency_name=dependency_name, extra_name=extra_name, pin=pin + ): + return True + return False + + +def _lock_proves_optional_git_dependency( + *, lock: dict[str, Any], dependency_name: str, extra_name: str, pin: GitRef +) -> bool: packages = lock.get("package") if not isinstance(packages, list): return False @@ -357,17 +370,59 @@ def _is_research_extra_only_dependency(repo_root: Path, pin: GitRef) -> bool: requires_dist = metadata.get("requires-dist") if isinstance(metadata, dict) else None if not isinstance(requires_dist, list): return False - for requirement in requires_dist: - if not isinstance(requirement, dict): - continue - if str(requirement.get("name", "")).lower() != dependency_name: - continue + matching = [ + requirement + for requirement in requires_dist + if isinstance(requirement, dict) + and str(requirement.get("name", "")).lower() == dependency_name + ] + if not matching: + return False + for requirement in matching: marker = str(requirement.get("marker", "")).strip() - research_marker = bool(re.fullmatch(r"\(?\s*extra\s*==\s*(['\"])research\1\s*\)?", marker)) + extra_marker = bool( + re.fullmatch(rf"\(?\s*extra\s*==\s*(['\"]){re.escape(extra_name)}\1\s*\)?", marker) + ) git = str(requirement.get("git", "")) - if research_marker and f"QuantStrategyLab/{pin.repo}.git" in git: - return True - return False + if not extra_marker or f"QuantStrategyLab/{pin.repo}.git" not in git: + return False + return True + + +_MIRROR_REQUIREMENT_RE = re.compile( + r"[A-Za-z0-9_.-]+\s+@\s+git\+https://github\.com/QuantStrategyLab/" + r"[A-Za-z0-9_.-]+\.git@[0-9a-f]{40}" +) + + +def _mirror_entries(path: Path, issues: list[str]) -> list[str]: + entries: list[str] = [] + for index, raw_line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + if not _MIRROR_REQUIREMENT_RE.fullmatch(line): + issues.append(f"invalid qsl pin mirror entry in {path.name}:{index}") + continue + entries.append(line) + return entries + + +def _validate_live_constraint_mirror( + *, repo_root: Path, constraint_path: Path, known_repos: set[str], issues: list[str] +) -> bool: + pins_path = repo_root / "qsl-pins.txt" + if not pins_path.exists(): + return False + constraint_entries = _mirror_entries(constraint_path, issues) + pin_entries = _mirror_entries(pins_path, issues) + for entry in constraint_entries + pin_entries: + match = re.search(r"QuantStrategyLab/([^/]+)\.git@", entry) + if match and match.group(1) not in known_repos: + issues.append(f"unmanaged qsl pin mirror entry in {constraint_path.name}: {match.group(1)}") + if constraint_entries != pin_entries: + issues.append(f"qsl pin mirror mismatch: {constraint_path.name} vs {pins_path.name}") + return True def _extract_git_refs(path: Path) -> list[GitRef]: @@ -535,8 +590,18 @@ def _check_current( lock_path = repo_root / "uv.lock" lock_refs = _extract_git_refs(lock_path) actual_refs = manifest_refs + lock_refs + mirror_files: set[Path] = set() for path in legacy_files: refs = _extract_git_refs(repo_root / path) + if repo_root.name == "QuantPlatformKit" and path.name in live_constraint_files and _validate_live_constraint_mirror( + repo_root=repo_root, + constraint_path=repo_root / path, + known_repos=managed_repos, + issues=issues, + ): + mirror_files.add(path) + if path in mirror_files: + continue if path.name in live_constraint_files: actual_refs.extend(refs) elif qsl_cfg["allow_legacy"]: