diff --git a/.github/workflows/test_actions.yml b/.github/workflows/test_actions.yml index 8e1f79a..52e4a18 100644 --- a/.github/workflows/test_actions.yml +++ b/.github/workflows/test_actions.yml @@ -254,6 +254,63 @@ jobs: project-directory: test-project expected-version: 1.0.2.dev1 + test_check_project_links: + name: Test check-project-links + runs-on: ubuntu-latest + steps: + - name: Check out repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Create project with a valid link + run: | + mkdir -p test-project + cat > test-project/pyproject.toml <<'EOF' + [project] + name = "test-project" + version = "0.1.0" + description = "Test project with a valid URL" + authors = [{ name = "NI" }] + readme = "README.md" + requires-python = ">=3.9" + + [project.urls] + Homepage = "https://github.com/ni/python-actions" + Documentation = "https://github.com/ni/python-actions/blob/main/README.md" + EOF + shell: bash + - name: Check valid project links + uses: ./check-project-links + with: + project-directory: test-project + - name: Create project with a missing link + run: | + mkdir -p failing-project + cat > failing-project/pyproject.toml <<'EOF' + [project] + name = "failing-project" + version = "0.1.0" + description = "Test project with a 404 URL" + authors = [{ name = "NI" }] + readme = "README.md" + requires-python = ">=3.9" + + [project.urls] + Broken = "https://github.com/ni/project-that-does-not-exist" + EOF + shell: bash + - name: Check missing project link (expected to fail) + id: expected-failure + continue-on-error: true + uses: ./check-project-links + with: + project-directory: failing-project + - name: Error if the previous step didn't fail + if: steps.expected-failure.outcome != 'failure' + run: | + echo "::error title=Test Failure::The previous step did not fail as expected." + exit 1 + test_analyze_project: name: Test analyze-project runs-on: ${{ matrix.os }} @@ -454,6 +511,7 @@ jobs: test_setup_poetry_no_cache, test_check_project_version, test_update_project_version, + test_check_project_links, test_analyze_project, test_analyze_project_repo_root, ] diff --git a/README.md b/README.md index b134f3b..f760978 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,9 @@ - [`ni/python-actions/check-project-version`](check-project-version): uses Poetry to get the version of a Python project and checks that it matches an expected version. Publish workflows can use this to verify that the release tag matches the version number in `pyproject.toml`. +- [`ni/python-actions/check-project-links`](check-project-links): scans project `pyproject.toml` files + for URLs, writes the discovered links to a temporary file, and validates each URL in Docker while + failing only on `4xx` responses. - [`ni/python-actions/update-project-version`](update-project-version): uses Poetry to update the version of a Python project and creates a pull request to modify its `pyproject.toml` file. Publish workflows can use this to update the version in `pyproject.toml` for the next build. diff --git a/check-project-links/README.md b/check-project-links/README.md new file mode 100644 index 0000000..6a03ff7 --- /dev/null +++ b/check-project-links/README.md @@ -0,0 +1,55 @@ +# `ni/python-actions/check-project-links` + +This action searches a project tree for `pyproject.toml` files, extracts `https://` links from those files, and validates each discovered URL by making an HTTP request in a Docker container. It fails only when a checked URL returns a `4xx` response. `2xx` and `3xx` responses are treated as successful and logged without failing the action. + +## Inputs + +### `project-directory` + +Path to the directory containing one or more `pyproject.toml` files. + +Default: `${{ github.workspace }}` + +### `allowed-domains` + +Comma-separated list of trusted hostnames or domains to validate. Supports wildcards like `*.readthedocs.io`. + +Default: `github.com,ni.github.io,*.readthedocs.io` + +### `docker-image` + +Docker image used to perform the HTTP requests. + +Default: `curlimages/curl:8.22.0@sha256:58adaa4e8dca9c988bae2aba4ab3434a0bb2da16bbe3f92dec39ec7785166777` + +> [!NOTE] +> The action default uses a full digest SHA, though this is not required. +## Examples + +> [!NOTE] +> These examples use `@v0`, but pinning to a commit hash or full release tag is recommended for +> build reproducibility and security. + + +```yaml +steps: + - uses: actions/checkout@v0 + + - name: Check project links + uses: ni/python-actions/check-project-links@v0 + with: + project-directory: . + docker-image: curlimages/curl:8.22.0 +``` + +## Behavior + +- Uses Python directory walk + regex search to identify `https://` links in `pyproject.toml` files under the provided path. +- Extracts `https://` substrings (including from comments) and filters them by the configured allowed domains before validation. +- Drops any URL whose hostname is `localhost`, a local loopback address, or any literal IP address before validation. +- Deduplicates the list of URLs and writes them to a temporary file. +- Validates each URL with the configured Docker image. +- Fails immediately if `docker` is not installed or available on `PATH`. +- Logs `2xx` and `3xx` responses as passing. +- Fails the action only when a URL returns a `4xx` status code. +- Other status codes are considered a warning. diff --git a/check-project-links/_find_project_links.py b/check-project-links/_find_project_links.py new file mode 100644 index 0000000..5b99398 --- /dev/null +++ b/check-project-links/_find_project_links.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 + +import argparse +import ipaddress +import os +import re +import sys +from urllib.parse import urlsplit + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Collect trusted project URLs from pyproject.toml files." + ) + parser.add_argument( + "project_directory", help="Directory containing project pyproject.toml files." + ) + parser.add_argument("allowed_domains", help="Comma-separated list of allowed domains.") + parser.add_argument("output_path", help="Path to write discovered URLs.") + return parser.parse_args() + + +def _is_allowed(hostname: str, allowed: set[str]) -> bool: + """Check if the given hostname is allowed based on the provided set of allowed domains. + + >>> _is_allowed('example.com', {'example.com'}) + True + + >>> _is_allowed('sub.example.com', {'example.com'}) + False + + >>> _is_allowed('sub.example.com', {'*.example.com'}) + True + """ + if not hostname: + return False + + host = hostname.lower().rstrip(".") + if host == "localhost" or host.endswith(".localhost"): + return False + + try: + ip = ipaddress.ip_address(host) + except ValueError: + ip = None + + if ip is not None: + return False + + if host in allowed: + return True + + if any( + host.endswith(f'.{domain.lstrip(".*")}') for domain in allowed if domain.startswith("*.") + ): + return True + + return False + + +def _safe_under(base_dir: str, candidate_path: str) -> str: + """Return a canonical path that stays under base_dir, or raise ValueError.""" + safe_base = os.path.realpath(base_dir) + safe_candidate = os.path.realpath(candidate_path) + + try: + if os.path.commonpath([safe_base, safe_candidate]) != safe_base: + raise ValueError(f"Path escapes base directory: {candidate_path!r}") + except ValueError as exc: + raise ValueError(f"Path escapes base directory: {candidate_path!r}") from exc + + return safe_candidate + + +_URL_RE = re.compile(r'(https://.+?)(?:"|\s)') + + +def _extract_links_from_line(line: str) -> list[str]: + """Extract URLs from a single line, stopping before a quote or whitespace.""" + matches: list[str] = [] + for match in re.finditer(_URL_RE, line): + url = match.group(1) + if url: + matches.append(url) + return matches + + +def _find_project_links(manifest_path: str, results: set[str], allowed: set[str]) -> None: + """Find URLs within a pyproject.toml file without parsing TOML.""" + try: + with open(manifest_path, "r", encoding="utf-8") as manifest_file: + for line in manifest_file: + for url in _extract_links_from_line(line): + host = urlsplit(url).hostname + if host and _is_allowed(host, allowed): + results.add(url) + except OSError: + print(f"Warning: Failed to read pyproject.toml at {manifest_path}", file=sys.stderr) + + +def _main() -> int: + args = _parse_args() + safe_project_directory = _safe_under(os.getcwd(), args.project_directory) + allowed_domains = args.allowed_domains + output_path = args.output_path + + allowed: set[str] = set() + for raw_domain in allowed_domains.split(","): + domain = raw_domain.strip().lower().rstrip(".") + if domain: + allowed.add(domain) + + results: set[str] = set() + + for root, _, files in os.walk(safe_project_directory): + for file_name in files: + if file_name != "pyproject.toml": + continue + + manifest_path = _safe_under(safe_project_directory, os.path.join(root, file_name)) + _find_project_links(manifest_path, results, allowed) + + output_directory = os.path.dirname(output_path) + if output_directory: + os.makedirs(output_directory, exist_ok=True) + + with open(output_path, "w", encoding="utf-8") as output_file: + for url in sorted(results): + output_file.write(f"{url}\n") + + return 0 + + +if __name__ == "__main__": + sys.exit(_main()) diff --git a/check-project-links/action.yml b/check-project-links/action.yml new file mode 100644 index 0000000..2055a9c --- /dev/null +++ b/check-project-links/action.yml @@ -0,0 +1,83 @@ +name: Check project links +description: Find URLs referenced in pyproject.toml files and fail only when a response is 4xx. + +inputs: + project-directory: + description: Path to the directory containing pyproject.toml files. + default: ${{ github.workspace }} + allowed-domains: + description: Comma-separated list of trusted hostnames or domains to validate. Supports wildcards like *.readthedocs.io. + default: github.com,ni.github.io,*.readthedocs.io + docker-image: + description: Docker image used to validate each discovered URL. + default: curlimages/curl:8.22.0@sha256:58adaa4e8dca9c988bae2aba4ab3434a0bb2da16bbe3f92dec39ec7785166777 + +runs: + using: composite + steps: + - name: Check project links + id: check-project-links + shell: bash + env: + PROJECT_DIRECTORY: ${{ inputs.project-directory }} + ALLOWED_DOMAINS: ${{ inputs.allowed-domains }} + DOCKER_IMAGE: ${{ inputs.docker-image }} + run: | + set -euo pipefail + + if [ ! -d "$PROJECT_DIRECTORY" ]; then + echo "::error title=Check Project Links Error::Project directory '$PROJECT_DIRECTORY' does not exist." + exit 1 + fi + + if ! command -v docker >/dev/null 2>&1; then + echo "::error title=Check Project Links Error::docker is not available. Install Docker or add a pre-step that installs it before using this action." + exit 1 + fi + + link_file="$(mktemp -p "$RUNNER_TEMP")" + cleanup() { + rm -f "$link_file" + } + trap cleanup EXIT + + chmod 644 "$link_file" # make the file world-readable so that the unprivileged user in the docker container can read it + + python3 "$GITHUB_ACTION_PATH/_find_project_links.py" "$PROJECT_DIRECTORY" "$ALLOWED_DOMAINS" "$link_file" + + if [ ! -s "$link_file" ]; then + echo "No trusted project links found under $PROJECT_DIRECTORY." + exit 0 + fi + + echo "Found $(wc -l < "$link_file") unique trusted links:" + cat "$link_file" + + unprivileged_user=100:100 # this matches the default, but we want to be explicit about it + docker run -u "${unprivileged_user}" --rm \ + -v "$link_file:/tmp/project_links.txt:ro" \ + "$DOCKER_IMAGE" \ + sh -ec ' + failed=0 + while IFS= read -r url; do + [ -n "$url" ] || continue + status=$(curl -L -sS --connect-timeout 5 --max-time 20 -o /tmp/link_body -w "%{http_code}" "$url" || true) + case "$status" in + 2??|3??) + echo "PASS $url -> $status" + ;; + 4??) + echo "FAIL $url -> $status" + failed=1 + ;; + *) + echo "WARN $url -> $status" + ;; + esac + done < /tmp/project_links.txt + exit "$failed" + ' + + # `docker run` failures already cause this step to fail due to `set -e`. + + echo "Success: No checked project links returned 4xx responses."