From e3c65689bbe949e1434e1e2a535023f43b4a76ba Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Wed, 2 Sep 2026 09:49:57 -0500 Subject: [PATCH 01/36] copilot: first draft --- README.md | 3 ++ check-project-links/README.md | 40 +++++++++++++++++ check-project-links/action.yml | 78 ++++++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+) create mode 100644 check-project-links/README.md create mode 100644 check-project-links/action.yml diff --git a/README.md b/README.md index b134f3b..f760978 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,9 @@ - [`ni/python-actions/check-project-version`](check-project-version): uses Poetry to get the version of a Python project and checks that it matches an expected version. Publish workflows can use this to verify that the release tag matches the version number in `pyproject.toml`. +- [`ni/python-actions/check-project-links`](check-project-links): scans project `pyproject.toml` files + for URLs, writes the discovered links to a temporary file, and validates each URL in Docker while + failing only on `4xx` responses. - [`ni/python-actions/update-project-version`](update-project-version): uses Poetry to update the version of a Python project and creates a pull request to modify its `pyproject.toml` file. Publish workflows can use this to update the version in `pyproject.toml` for the next build. diff --git a/check-project-links/README.md b/check-project-links/README.md new file mode 100644 index 0000000..404b425 --- /dev/null +++ b/check-project-links/README.md @@ -0,0 +1,40 @@ +# `ni/python-actions/check-project-links` + +This action searches a project tree for `pyproject.toml` files, extracts `https://` links from those files, and validates each discovered URL by making an HTTP request in a Docker container. It fails only when a checked URL returns a `4xx` response. `2xx` and `3xx` responses are treated as successful and logged without failing the action. + +## Inputs + +### `project-directory` + +Path to the directory containing one or more `pyproject.toml` files. + +Default: `${{ github.workspace }}` + +### `docker-image` + +Docker image used to perform the HTTP requests. + +Default: `curlimages/curl:8.22.0` + +## Example + +```yaml +steps: + - uses: actions/checkout@v4 + + - name: Check project links + uses: ni/python-actions/check-project-links@v1 + with: + project-directory: . + docker-image: curlimages/curl:8.22.0 +``` + +## Behavior + +- Uses `rg` to locate `pyproject.toml` files beneath the configured project directory. +- Extracts every `https://...` URL that ends at the first whitespace character. +- Deduplicates the list of URLs and writes them to a temporary file. +- Validates each URL with the configured Docker image. +- Logs `2xx` and `3xx` responses as passing. +- Fails the action only when a URL returns a `4xx` status code. +- Other status codes are considered a warning. diff --git a/check-project-links/action.yml b/check-project-links/action.yml new file mode 100644 index 0000000..6c8dd5a --- /dev/null +++ b/check-project-links/action.yml @@ -0,0 +1,78 @@ +name: Check project links +description: Find URLs referenced in pyproject.toml files and fail only when a response is 4xx. + +inputs: + project-directory: + description: Path to the directory containing pyproject.toml files. + default: ${{ github.workspace }} + docker-image: + description: Docker image used to validate each discovered URL. + default: curlimages/curl:8.22.0@sha256:58adaa4e8dca9c988bae2aba4ab3434a0bb2da16bbe3f92dec39ec7785166777 + +runs: + using: composite + steps: + - name: Check project links + shell: bash + run: | + set -u + + project_directory="${{ inputs.project-directory }}" + docker_image="${{ inputs.docker-image }}" + + if [ ! -d "$project_directory" ]; then + echo "::error title=Check Project Links Error::Project directory '$project_directory' does not exist." + exit 1 + fi + + link_file="$(mktemp)" + cleanup() { + rm -f "$link_file" + } + trap cleanup EXIT + + while IFS= read -r manifest; do + if [ -n "$manifest" ]; then + rg -o 'https://[^[:space:]]+' "$manifest" || true + rg 'http://[^[:space:]]+' "$manifest" && echo "::warning title=Check Project Links Warning::Found insecure HTTP link in $manifest. Consider using HTTPS instead. Ignoring linke." || true + fi + done < <(rg --files -g 'pyproject.toml' "$project_directory") | sort -u > "$link_file" + + if [ ! -s "$link_file" ]; then + echo "No project links found under $project_directory." + exit 0 + fi + + echo "Found $(wc -l < "$link_file") unique links:" + cat "$link_file" + + docker run --rm \ + -v "$link_file:/tmp/project_links.txt:ro" \ + "$docker_image" \ + sh -ec ' + failed=0 + while IFS= read -r url; do + [ -n "$url" ] || continue + status=$(curl -L -sS -o /tmp/link_body -w "%{http_code}" "$url" || true) + case "$status" in + 2??|3??) + echo "PASS $url -> $status" + ;; + 4??) + echo "FAIL $url -> $status" + failed=1 + ;; + *) + echo "WARN $url -> $status" + ;; + esac + done < /tmp/project_links.txt + exit "$failed" + ' + + status=$? + if [ "$status" -ne 0 ]; then + exit 1 + fi + + echo "Success: No checked project links returned 4xx responses." From ab0adce0540554fcf23660d1b2cd0dc3e52b7878 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Wed, 2 Sep 2026 09:58:33 -0500 Subject: [PATCH 02/36] copilot: setup tests --- .github/workflows/test_actions.yml | 58 ++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/.github/workflows/test_actions.yml b/.github/workflows/test_actions.yml index 8e1f79a..52e4a18 100644 --- a/.github/workflows/test_actions.yml +++ b/.github/workflows/test_actions.yml @@ -254,6 +254,63 @@ jobs: project-directory: test-project expected-version: 1.0.2.dev1 + test_check_project_links: + name: Test check-project-links + runs-on: ubuntu-latest + steps: + - name: Check out repo + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Create project with a valid link + run: | + mkdir -p test-project + cat > test-project/pyproject.toml <<'EOF' + [project] + name = "test-project" + version = "0.1.0" + description = "Test project with a valid URL" + authors = [{ name = "NI" }] + readme = "README.md" + requires-python = ">=3.9" + + [project.urls] + Homepage = "https://github.com/ni/python-actions" + Documentation = "https://github.com/ni/python-actions/blob/main/README.md" + EOF + shell: bash + - name: Check valid project links + uses: ./check-project-links + with: + project-directory: test-project + - name: Create project with a missing link + run: | + mkdir -p failing-project + cat > failing-project/pyproject.toml <<'EOF' + [project] + name = "failing-project" + version = "0.1.0" + description = "Test project with a 404 URL" + authors = [{ name = "NI" }] + readme = "README.md" + requires-python = ">=3.9" + + [project.urls] + Broken = "https://github.com/ni/project-that-does-not-exist" + EOF + shell: bash + - name: Check missing project link (expected to fail) + id: expected-failure + continue-on-error: true + uses: ./check-project-links + with: + project-directory: failing-project + - name: Error if the previous step didn't fail + if: steps.expected-failure.outcome != 'failure' + run: | + echo "::error title=Test Failure::The previous step did not fail as expected." + exit 1 + test_analyze_project: name: Test analyze-project runs-on: ${{ matrix.os }} @@ -454,6 +511,7 @@ jobs: test_setup_poetry_no_cache, test_check_project_version, test_update_project_version, + test_check_project_links, test_analyze_project, test_analyze_project_repo_root, ] From 705a350f36b6de31673883567251d8325258b8a5 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Wed, 2 Sep 2026 09:58:41 -0500 Subject: [PATCH 03/36] fix zizmor issues --- check-project-links/action.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/check-project-links/action.yml b/check-project-links/action.yml index 6c8dd5a..f9d4b4e 100644 --- a/check-project-links/action.yml +++ b/check-project-links/action.yml @@ -14,11 +14,14 @@ runs: steps: - name: Check project links shell: bash + env: + PROJECT_DIRECTORY: ${{ inputs.project-directory }} + DOCKER_IMAGE: ${{ inputs.docker-image }} run: | set -u - project_directory="${{ inputs.project-directory }}" - docker_image="${{ inputs.docker-image }}" + project_directory="$PROJECT_DIRECTORY" + docker_image="$DOCKER_IMAGE" if [ ! -d "$project_directory" ]; then echo "::error title=Check Project Links Error::Project directory '$project_directory' does not exist." From 30bef4ffd877699f2423adc544faa43df7cdba0c Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Wed, 2 Sep 2026 09:58:54 -0500 Subject: [PATCH 04/36] use @v0 in doc and add disclaimer --- check-project-links/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/check-project-links/README.md b/check-project-links/README.md index 404b425..c1516ec 100644 --- a/check-project-links/README.md +++ b/check-project-links/README.md @@ -16,11 +16,16 @@ Docker image used to perform the HTTP requests. Default: `curlimages/curl:8.22.0` -## Example +## Examples + +> [!NOTE] +> These examples use `@v0`, but pinning to a commit hash or full release tag is recommended for +> build reproducibility and security. + ```yaml steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v0 - name: Check project links uses: ni/python-actions/check-project-links@v1 From 6116fb7285471ad8bff2f24267740191274fbf75 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Wed, 2 Sep 2026 10:01:13 -0500 Subject: [PATCH 05/36] don't care about the full link for http warning --- check-project-links/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check-project-links/action.yml b/check-project-links/action.yml index f9d4b4e..7f57c7e 100644 --- a/check-project-links/action.yml +++ b/check-project-links/action.yml @@ -37,7 +37,7 @@ runs: while IFS= read -r manifest; do if [ -n "$manifest" ]; then rg -o 'https://[^[:space:]]+' "$manifest" || true - rg 'http://[^[:space:]]+' "$manifest" && echo "::warning title=Check Project Links Warning::Found insecure HTTP link in $manifest. Consider using HTTPS instead. Ignoring linke." || true + rg 'http://' "$manifest" && echo "::warning title=Check Project Links Warning::Found insecure HTTP link in $manifest. Consider using HTTPS instead. Ignoring linke." || true fi done < <(rg --files -g 'pyproject.toml' "$project_directory") | sort -u > "$link_file" From fd83fa7ea2e25c87689e8198ad976b5aba893dba Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Wed, 2 Sep 2026 10:02:57 -0500 Subject: [PATCH 06/36] also handle quote ended links --- check-project-links/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check-project-links/action.yml b/check-project-links/action.yml index 7f57c7e..7ea2f91 100644 --- a/check-project-links/action.yml +++ b/check-project-links/action.yml @@ -36,7 +36,7 @@ runs: while IFS= read -r manifest; do if [ -n "$manifest" ]; then - rg -o 'https://[^[:space:]]+' "$manifest" || true + rg -o "https://[^\"[:space:]]+" "$manifest" || true rg 'http://' "$manifest" && echo "::warning title=Check Project Links Warning::Found insecure HTTP link in $manifest. Consider using HTTPS instead. Ignoring linke." || true fi done < <(rg --files -g 'pyproject.toml' "$project_directory") | sort -u > "$link_file" From 44028acfd944a2f45e7e589f7ccf5f3ba81a7748 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Wed, 2 Sep 2026 10:09:52 -0500 Subject: [PATCH 07/36] fix copilot suggested issues --- check-project-links/README.md | 3 +++ check-project-links/action.yml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/check-project-links/README.md b/check-project-links/README.md index c1516ec..0df0229 100644 --- a/check-project-links/README.md +++ b/check-project-links/README.md @@ -16,6 +16,9 @@ Docker image used to perform the HTTP requests. Default: `curlimages/curl:8.22.0` +> [!NOTE] +> The action default uses a full digest sha. Although the action does not require this. + ## Examples > [!NOTE] diff --git a/check-project-links/action.yml b/check-project-links/action.yml index 7ea2f91..9b92cd6 100644 --- a/check-project-links/action.yml +++ b/check-project-links/action.yml @@ -37,7 +37,7 @@ runs: while IFS= read -r manifest; do if [ -n "$manifest" ]; then rg -o "https://[^\"[:space:]]+" "$manifest" || true - rg 'http://' "$manifest" && echo "::warning title=Check Project Links Warning::Found insecure HTTP link in $manifest. Consider using HTTPS instead. Ignoring linke." || true + rg 'http://' "$manifest" && echo "::warning title=Check Project Links Warning::Found insecure HTTP link in $manifest. Consider using HTTPS instead. Ignoring link." || true fi done < <(rg --files -g 'pyproject.toml' "$project_directory") | sort -u > "$link_file" From 882ba28d2480a737159bd3507c9de57899cc55a5 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Wed, 2 Sep 2026 10:24:59 -0500 Subject: [PATCH 08/36] add trusted domains input and limit to only checking https links at those domains --- check-project-links/action.yml | 80 +++++++++++++++++++++++++++++++--- 1 file changed, 74 insertions(+), 6 deletions(-) diff --git a/check-project-links/action.yml b/check-project-links/action.yml index 9b92cd6..d2832c7 100644 --- a/check-project-links/action.yml +++ b/check-project-links/action.yml @@ -5,6 +5,9 @@ inputs: project-directory: description: Path to the directory containing pyproject.toml files. default: ${{ github.workspace }} + allowed-domains: + description: Comma-separated list of trusted hostnames or domains to validate. Supports wildcards like *.readthedocs.io. + default: github.com,ni.github.io,*.readthedocs.io docker-image: description: Docker image used to validate each discovered URL. default: curlimages/curl:8.22.0@sha256:58adaa4e8dca9c988bae2aba4ab3434a0bb2da16bbe3f92dec39ec7785166777 @@ -16,11 +19,13 @@ runs: shell: bash env: PROJECT_DIRECTORY: ${{ inputs.project-directory }} + ALLOWED_DOMAINS: ${{ inputs.allowed-domains }} DOCKER_IMAGE: ${{ inputs.docker-image }} run: | set -u project_directory="$PROJECT_DIRECTORY" + allowed_domains="$ALLOWED_DOMAINS" docker_image="$DOCKER_IMAGE" if [ ! -d "$project_directory" ]; then @@ -28,25 +33,88 @@ runs: exit 1 fi + manifest_links="$(mktemp)" link_file="$(mktemp)" cleanup() { - rm -f "$link_file" + rm -f "$manifest_links" "$link_file" } trap cleanup EXIT while IFS= read -r manifest; do if [ -n "$manifest" ]; then - rg -o "https://[^\"[:space:]]+" "$manifest" || true - rg 'http://' "$manifest" && echo "::warning title=Check Project Links Warning::Found insecure HTTP link in $manifest. Consider using HTTPS instead. Ignoring link." || true + rg -o "https\?://[^\"[:space:]]+" "$manifest" || true + if rg 'http://' "$manifest" >/dev/null; then + echo "::warning title=Check Project Links Warning::Found insecure HTTP link in $manifest. Consider using HTTPS instead. Ignoring link." + fi fi - done < <(rg --files -g 'pyproject.toml' "$project_directory") | sort -u > "$link_file" + done < <(rg --files -g 'pyproject.toml' "$project_directory") | sort -u > "$manifest_links" + + : > "$link_file" + while IFS= read -r url; do + [ -n "$url" ] || continue + + if printf '%s\n' "$url" | grep -q 'http://'; then + echo "::warning title=Check Project Links Warning::Found insecure HTTP link in $url. Ignoring link." + continue + fi + + host="${url#*://}" + host="${host%%/*}" + host="${host%%:*}" + host="${host,,}" + host="${host%.}" + + if printf '%s\n' "$host" | grep -Eq '^(localhost|.*\.localhost)$'; then + continue + fi + if printf '%s\n' "$host" | grep -Eq '^(127\.|0\.0\.0\.0|::1|169\.254\.|metadata(\.google\.internal)?)$'; then + continue + fi + if printf '%s\n' "$host" | grep -Eq '^(10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[0-1])\.|169\.254\.|fc[0-9a-fA-F:]+|fe[89ab][0-9a-fA-F:]+)$'; then + continue + fi + + allowed=0 + IFS=',' read -ra domains <<< "$allowed_domains" + for domain in "${domains[@]}"; do + domain="${domain//[[:space:]]/}" + [ -n "$domain" ] || continue + domain="${domain,,}" + domain="${domain%.}" + + if [ "${domain:0:2}" = '*.' ]; then + suffix="${domain#*.}" + if printf '%s\n' "$host" | grep -Eq "^([A-Za-z0-9-]+\.)*${suffix//./\\.}$"; then + allowed=1 + break + fi + else + if [ "$host" = "$domain" ] || printf '%s\n' "$host" | grep -Eq "^([A-Za-z0-9-]+\.)+${domain//./\\.}$"; then + allowed=1 + break + fi + fi + done + + if [ "$allowed" -eq 1 ]; then + printf '%s\n' "$url" >> "$link_file" + fi + done < "$manifest_links" + + # Explicitly drop insecure HTTP links before validation. + grep -v 'http://' "$link_file" > "${link_file}.https" || true + if [ -s "${link_file}.https" ]; then + mv "${link_file}.https" "$link_file" + else + : > "$link_file" + fi if [ ! -s "$link_file" ]; then - echo "No project links found under $project_directory." + echo "No trusted project links found under $project_directory." exit 0 fi - echo "Found $(wc -l < "$link_file") unique links:" + echo "Found $(wc -l < "$link_file") unique trusted links:" cat "$link_file" docker run --rm \ From c04646dbb925fec6b4752ca6b0f3d0bda70966cd Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Wed, 2 Sep 2026 10:32:57 -0500 Subject: [PATCH 09/36] add checks for rg and docker commands --- check-project-links/README.md | 4 +++- check-project-links/action.yml | 34 ++++++++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/check-project-links/README.md b/check-project-links/README.md index 0df0229..d5cce48 100644 --- a/check-project-links/README.md +++ b/check-project-links/README.md @@ -39,10 +39,12 @@ steps: ## Behavior -- Uses `rg` to locate `pyproject.toml` files beneath the configured project directory. +- Uses `rg` to locate `pyproject.toml` files beneath the configured project directory when available. +- Falls back to `grep` with a warning if `rg` is not installed; consider adding a pre-step to install `rg` for faster runtime. - Extracts every `https://...` URL that ends at the first whitespace character. - Deduplicates the list of URLs and writes them to a temporary file. - Validates each URL with the configured Docker image. +- Fails immediately if `docker` is not installed or available on `PATH`. - Logs `2xx` and `3xx` responses as passing. - Fails the action only when a URL returns a `4xx` status code. - Other status codes are considered a warning. diff --git a/check-project-links/action.yml b/check-project-links/action.yml index d2832c7..cfa972e 100644 --- a/check-project-links/action.yml +++ b/check-project-links/action.yml @@ -24,6 +24,12 @@ runs: run: | set -u + + if ! command -v docker >/dev/null 2>&1; then + echo "::error title=Check Project Links Error::docker is not available. Install Docker or add a pre-step that installs it before using this action." + exit 1 + fi + project_directory="$PROJECT_DIRECTORY" allowed_domains="$ALLOWED_DOMAINS" docker_image="$DOCKER_IMAGE" @@ -40,14 +46,34 @@ runs: } trap cleanup EXIT + if command -v rg >/dev/null 2>&1; then + rg_available=1 + else + rg_available=0 + echo "::warning title=Check Project Links Warning::rg is not available; falling back to grep. Consider adding a pre-step to install rg for faster runtime." + fi + + if [ "$rg_available" -eq 1 ]; then + manifest_list_cmd="rg --files -g 'pyproject.toml' \"$project_directory\"" + else + manifest_list_cmd="find \"$project_directory\" -type f -name 'pyproject.toml'" + fi + while IFS= read -r manifest; do if [ -n "$manifest" ]; then - rg -o "https\?://[^\"[:space:]]+" "$manifest" || true - if rg 'http://' "$manifest" >/dev/null; then - echo "::warning title=Check Project Links Warning::Found insecure HTTP link in $manifest. Consider using HTTPS instead. Ignoring link." + if [ "$rg_available" -eq 1 ]; then + rg -o "https\?://[^\"[:space:]]+" "$manifest" || true + if rg 'http://' "$manifest" >/dev/null 2>&1; then + echo "::warning title=Check Project Links Warning::Found insecure HTTP link in $manifest. Consider using HTTPS instead. Ignoring link." + fi + else + grep -Eo 'https?://[^[:space:]\"]+' "$manifest" || true + if grep -Eq 'http://' "$manifest"; then + echo "::warning title=Check Project Links Warning::Found insecure HTTP link in $manifest. Consider using HTTPS instead. Ignoring link." + fi fi fi - done < <(rg --files -g 'pyproject.toml' "$project_directory") | sort -u > "$manifest_links" + done < <(eval "$manifest_list_cmd") | sort -u > "$manifest_links" : > "$link_file" while IFS= read -r url; do From e3d97a18cbb57e060419e232a4b1f8d894e820a6 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Wed, 2 Sep 2026 10:33:12 -0500 Subject: [PATCH 10/36] whitespace --- check-project-links/action.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/check-project-links/action.yml b/check-project-links/action.yml index cfa972e..9931fa6 100644 --- a/check-project-links/action.yml +++ b/check-project-links/action.yml @@ -24,7 +24,6 @@ runs: run: | set -u - if ! command -v docker >/dev/null 2>&1; then echo "::error title=Check Project Links Error::docker is not available. Install Docker or add a pre-step that installs it before using this action." exit 1 From 35617bd76e6da820b5798198ebe3e4c0be4cd0c5 Mon Sep 17 00:00:00 2001 From: mshafer-NI <23644905+mshafer-NI@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:43:33 -0500 Subject: [PATCH 11/36] Apply batched suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- check-project-links/README.md | 11 ++++++++--- check-project-links/action.yml | 24 +++++++++++------------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/check-project-links/README.md b/check-project-links/README.md index d5cce48..d0aff4d 100644 --- a/check-project-links/README.md +++ b/check-project-links/README.md @@ -10,15 +10,20 @@ Path to the directory containing one or more `pyproject.toml` files. Default: `${{ github.workspace }}` +### `allowed-domains` + +Comma-separated list of trusted hostnames or domains to validate. Supports wildcards like `*.readthedocs.io`. + +Default: `github.com,ni.github.io,*.readthedocs.io` + ### `docker-image` Docker image used to perform the HTTP requests. -Default: `curlimages/curl:8.22.0` +Default: `curlimages/curl:8.22.0@sha256:58adaa4e8dca9c988bae2aba4ab3434a0bb2da16bbe3f92dec39ec7785166777` > [!NOTE] -> The action default uses a full digest sha. Although the action does not require this. - +> The action default uses a full digest SHA, though this is not required. ## Examples > [!NOTE] diff --git a/check-project-links/action.yml b/check-project-links/action.yml index cfa972e..eed8a2a 100644 --- a/check-project-links/action.yml +++ b/check-project-links/action.yml @@ -42,7 +42,7 @@ runs: manifest_links="$(mktemp)" link_file="$(mktemp)" cleanup() { - rm -f "$manifest_links" "$link_file" + rm -f "$manifest_links" "$link_file" "${link_file}.https" } trap cleanup EXIT @@ -54,9 +54,9 @@ runs: fi if [ "$rg_available" -eq 1 ]; then - manifest_list_cmd="rg --files -g 'pyproject.toml' \"$project_directory\"" + manifest_list_cmd=(rg --files -g 'pyproject.toml' "$project_directory") else - manifest_list_cmd="find \"$project_directory\" -type f -name 'pyproject.toml'" + manifest_list_cmd=(find "$project_directory" -type f -name 'pyproject.toml') fi while IFS= read -r manifest; do @@ -73,7 +73,7 @@ runs: fi fi fi - done < <(eval "$manifest_list_cmd") | sort -u > "$manifest_links" + done < <("${manifest_list_cmd[@]}") | sort -u > "$manifest_links" : > "$link_file" while IFS= read -r url; do @@ -110,15 +110,13 @@ runs: if [ "${domain:0:2}" = '*.' ]; then suffix="${domain#*.}" - if printf '%s\n' "$host" | grep -Eq "^([A-Za-z0-9-]+\.)*${suffix//./\\.}$"; then - allowed=1 - break - fi + case "$host" in + "$suffix"|*."$suffix") allowed=1; break ;; + esac else - if [ "$host" = "$domain" ] || printf '%s\n' "$host" | grep -Eq "^([A-Za-z0-9-]+\.)+${domain//./\\.}$"; then - allowed=1 - break - fi + case "$host" in + "$domain"|*."$domain") allowed=1; break ;; + esac fi done @@ -150,7 +148,7 @@ runs: failed=0 while IFS= read -r url; do [ -n "$url" ] || continue - status=$(curl -L -sS -o /tmp/link_body -w "%{http_code}" "$url" || true) + status=$(curl -L -sS --connect-timeout 5 --max-time 20 -o /tmp/link_body -w "%{http_code}" "$url" || true) case "$status" in 2??|3??) echo "PASS $url -> $status" From c7b1ff9230da31ddd17fc8f1d946f990eea65cd8 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Wed, 2 Sep 2026 10:50:50 -0500 Subject: [PATCH 12/36] explicitly drop loopback and IP based hosts --- check-project-links/README.md | 1 + check-project-links/action.yml | 38 +++++++++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/check-project-links/README.md b/check-project-links/README.md index d0aff4d..464252b 100644 --- a/check-project-links/README.md +++ b/check-project-links/README.md @@ -47,6 +47,7 @@ steps: - Uses `rg` to locate `pyproject.toml` files beneath the configured project directory when available. - Falls back to `grep` with a warning if `rg` is not installed; consider adding a pre-step to install `rg` for faster runtime. - Extracts every `https://...` URL that ends at the first whitespace character. +- Drops any URL whose hostname is `localhost`, a local loopback address, or any literal IP address before validation. - Deduplicates the list of URLs and writes them to a temporary file. - Validates each URL with the configured Docker image. - Fails immediately if `docker` is not installed or available on `PATH`. diff --git a/check-project-links/action.yml b/check-project-links/action.yml index c2e7b8a..43761a0 100644 --- a/check-project-links/action.yml +++ b/check-project-links/action.yml @@ -41,7 +41,7 @@ runs: manifest_links="$(mktemp)" link_file="$(mktemp)" cleanup() { - rm -f "$manifest_links" "$link_file" "${link_file}.https" + rm -f "$manifest_links" "$link_file" "${link_file}.https" "${link_file}.filtered" } trap cleanup EXIT @@ -132,6 +132,42 @@ runs: : > "$link_file" fi + # Explicitly drop localhost and any IP-based hostname before validation. + : > "${link_file}.filtered" + while IFS= read -r url; do + [ -n "$url" ] || continue + + host="${url#*://}" + host="${host%%/*}" + host="${host%%:*}" + host="${host,,}" + host="${host%.}" + + if printf '%s\n' "$host" | grep -Eq '^(localhost|.*\.localhost)$'; then + continue + fi + if printf '%s\n' "$host" | grep -Eq '^[0-9]+(\.[0-9]+){3}$'; then + continue + fi + if printf '%s\n' "$host" | grep -Eq '^(::1|[0-9a-fA-F:]+)$'; then + continue + fi + if printf '%s\n' "$host" | grep -Eq '^(127\.|0\.0\.0\.0|169\.254\.|metadata(\.google\.internal)?)$'; then + continue + fi + if printf '%s\n' "$host" | grep -Eq '^(10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[0-1])\.|fc[0-9a-fA-F:]+|fe[89ab][0-9a-fA-F:]+)$'; then + continue + fi + + printf '%s\n' "$url" >> "${link_file}.filtered" + done < "$link_file" + + if [ -s "${link_file}.filtered" ]; then + mv "${link_file}.filtered" "$link_file" + else + : > "$link_file" + fi + if [ ! -s "$link_file" ]; then echo "No trusted project links found under $project_directory." exit 0 From bd73f6a99653f36a2450fcc90ecbf0cec497cc05 Mon Sep 17 00:00:00 2001 From: mshafer-NI <23644905+mshafer-NI@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:56:44 -0500 Subject: [PATCH 13/36] Apply batched suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- check-project-links/README.md | 4 ++-- check-project-links/action.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/check-project-links/README.md b/check-project-links/README.md index 464252b..8544d59 100644 --- a/check-project-links/README.md +++ b/check-project-links/README.md @@ -45,8 +45,8 @@ steps: ## Behavior - Uses `rg` to locate `pyproject.toml` files beneath the configured project directory when available. -- Falls back to `grep` with a warning if `rg` is not installed; consider adding a pre-step to install `rg` for faster runtime. -- Extracts every `https://...` URL that ends at the first whitespace character. +- Falls back to `find` (discovery) and `grep` (extraction) with a warning if `rg` is not installed; consider adding a pre-step to install `rg` for faster runtime. +- Extracts every `https://...` URL that ends at the first whitespace or double-quote character. - Drops any URL whose hostname is `localhost`, a local loopback address, or any literal IP address before validation. - Deduplicates the list of URLs and writes them to a temporary file. - Validates each URL with the configured Docker image. diff --git a/check-project-links/action.yml b/check-project-links/action.yml index 43761a0..0de2dae 100644 --- a/check-project-links/action.yml +++ b/check-project-links/action.yml @@ -61,12 +61,12 @@ runs: while IFS= read -r manifest; do if [ -n "$manifest" ]; then if [ "$rg_available" -eq 1 ]; then - rg -o "https\?://[^\"[:space:]]+" "$manifest" || true + rg -o 'https://[^"[:space:]]+' "$manifest" || true if rg 'http://' "$manifest" >/dev/null 2>&1; then echo "::warning title=Check Project Links Warning::Found insecure HTTP link in $manifest. Consider using HTTPS instead. Ignoring link." fi else - grep -Eo 'https?://[^[:space:]\"]+' "$manifest" || true + grep -Eo 'https://[^[:space:]\"]+' "$manifest" || true if grep -Eq 'http://' "$manifest"; then echo "::warning title=Check Project Links Warning::Found insecure HTTP link in $manifest. Consider using HTTPS instead. Ignoring link." fi From 8943bd40cafa70b460d1075575cbe250eb2371d4 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Wed, 2 Sep 2026 16:24:14 -0500 Subject: [PATCH 14/36] use existing env vars directly --- check-project-links/action.yml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/check-project-links/action.yml b/check-project-links/action.yml index 43761a0..e0a79a4 100644 --- a/check-project-links/action.yml +++ b/check-project-links/action.yml @@ -29,12 +29,8 @@ runs: exit 1 fi - project_directory="$PROJECT_DIRECTORY" - allowed_domains="$ALLOWED_DOMAINS" - docker_image="$DOCKER_IMAGE" - - if [ ! -d "$project_directory" ]; then - echo "::error title=Check Project Links Error::Project directory '$project_directory' does not exist." + if [ ! -d "$PROJECT_DIRECTORY" ]; then + echo "::error title=Check Project Links Error::Project directory '$PROJECT_DIRECTORY' does not exist." exit 1 fi @@ -53,9 +49,9 @@ runs: fi if [ "$rg_available" -eq 1 ]; then - manifest_list_cmd=(rg --files -g 'pyproject.toml' "$project_directory") + manifest_list_cmd=(rg --files -g 'pyproject.toml' "$PROJECT_DIRECTORY") else - manifest_list_cmd=(find "$project_directory" -type f -name 'pyproject.toml') + manifest_list_cmd=(find "$PROJECT_DIRECTORY" -type f -name 'pyproject.toml') fi while IFS= read -r manifest; do @@ -100,7 +96,7 @@ runs: fi allowed=0 - IFS=',' read -ra domains <<< "$allowed_domains" + IFS=',' read -ra domains <<< "$ALLOWED_DOMAINS" for domain in "${domains[@]}"; do domain="${domain//[[:space:]]/}" [ -n "$domain" ] || continue @@ -169,7 +165,7 @@ runs: fi if [ ! -s "$link_file" ]; then - echo "No trusted project links found under $project_directory." + echo "No trusted project links found under $PROJECT_DIRECTORY." exit 0 fi @@ -178,7 +174,7 @@ runs: docker run --rm \ -v "$link_file:/tmp/project_links.txt:ro" \ - "$docker_image" \ + "$DOCKER_IMAGE" \ sh -ec ' failed=0 while IFS= read -r url; do From b61454c82584755b5a4486713ccde9f6535a1881 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Wed, 2 Sep 2026 16:44:06 -0500 Subject: [PATCH 15/36] move it to a py file --- check-project-links/action.yml | 161 +++++----------------- check-project-links/find_project_links.py | 102 ++++++++++++++ 2 files changed, 133 insertions(+), 130 deletions(-) create mode 100644 check-project-links/find_project_links.py diff --git a/check-project-links/action.yml b/check-project-links/action.yml index e0a79a4..caaa8de 100644 --- a/check-project-links/action.yml +++ b/check-project-links/action.yml @@ -15,165 +15,66 @@ inputs: runs: using: composite steps: - - name: Check project links + - name: Get project links + id: get_project_links shell: bash env: PROJECT_DIRECTORY: ${{ inputs.project-directory }} ALLOWED_DOMAINS: ${{ inputs.allowed-domains }} - DOCKER_IMAGE: ${{ inputs.docker-image }} run: | set -u - if ! command -v docker >/dev/null 2>&1; then - echo "::error title=Check Project Links Error::docker is not available. Install Docker or add a pre-step that installs it before using this action." - exit 1 - fi - if [ ! -d "$PROJECT_DIRECTORY" ]; then echo "::error title=Check Project Links Error::Project directory '$PROJECT_DIRECTORY' does not exist." exit 1 fi - manifest_links="$(mktemp)" link_file="$(mktemp)" cleanup() { - rm -f "$manifest_links" "$link_file" "${link_file}.https" "${link_file}.filtered" + rm -f "$link_file" } trap cleanup EXIT - if command -v rg >/dev/null 2>&1; then - rg_available=1 - else - rg_available=0 - echo "::warning title=Check Project Links Warning::rg is not available; falling back to grep. Consider adding a pre-step to install rg for faster runtime." - fi + python3 "$GITHUB_ACTION_PATH/find_project_links.py" "$PROJECT_DIRECTORY" "$ALLOWED_DOMAINS" "$link_file" - if [ "$rg_available" -eq 1 ]; then - manifest_list_cmd=(rg --files -g 'pyproject.toml' "$PROJECT_DIRECTORY") - else - manifest_list_cmd=(find "$PROJECT_DIRECTORY" -type f -name 'pyproject.toml') + echo "link-file=$link_file" >> "$GITHUB_OUTPUT" + + if [ ! -s "$link_file" ]; then + echo "No trusted project links found under $PROJECT_DIRECTORY." + exit 0 fi - while IFS= read -r manifest; do - if [ -n "$manifest" ]; then - if [ "$rg_available" -eq 1 ]; then - rg -o "https\?://[^\"[:space:]]+" "$manifest" || true - if rg 'http://' "$manifest" >/dev/null 2>&1; then - echo "::warning title=Check Project Links Warning::Found insecure HTTP link in $manifest. Consider using HTTPS instead. Ignoring link." - fi - else - grep -Eo 'https?://[^[:space:]\"]+' "$manifest" || true - if grep -Eq 'http://' "$manifest"; then - echo "::warning title=Check Project Links Warning::Found insecure HTTP link in $manifest. Consider using HTTPS instead. Ignoring link." - fi - fi - fi - done < <("${manifest_list_cmd[@]}") | sort -u > "$manifest_links" - - : > "$link_file" - while IFS= read -r url; do - [ -n "$url" ] || continue - - if printf '%s\n' "$url" | grep -q 'http://'; then - echo "::warning title=Check Project Links Warning::Found insecure HTTP link in $url. Ignoring link." - continue - fi - - host="${url#*://}" - host="${host%%/*}" - host="${host%%:*}" - host="${host,,}" - host="${host%.}" - - if printf '%s\n' "$host" | grep -Eq '^(localhost|.*\.localhost)$'; then - continue - fi - if printf '%s\n' "$host" | grep -Eq '^(127\.|0\.0\.0\.0|::1|169\.254\.|metadata(\.google\.internal)?)$'; then - continue - fi - if printf '%s\n' "$host" | grep -Eq '^(10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[0-1])\.|169\.254\.|fc[0-9a-fA-F:]+|fe[89ab][0-9a-fA-F:]+)$'; then - continue - fi - - allowed=0 - IFS=',' read -ra domains <<< "$ALLOWED_DOMAINS" - for domain in "${domains[@]}"; do - domain="${domain//[[:space:]]/}" - [ -n "$domain" ] || continue - domain="${domain,,}" - domain="${domain%.}" - - if [ "${domain:0:2}" = '*.' ]; then - suffix="${domain#*.}" - case "$host" in - "$suffix"|*."$suffix") allowed=1; break ;; - esac - else - case "$host" in - "$domain"|*."$domain") allowed=1; break ;; - esac - fi - done - - if [ "$allowed" -eq 1 ]; then - printf '%s\n' "$url" >> "$link_file" - fi - done < "$manifest_links" - - # Explicitly drop insecure HTTP links before validation. - grep -v 'http://' "$link_file" > "${link_file}.https" || true - if [ -s "${link_file}.https" ]; then - mv "${link_file}.https" "$link_file" - else - : > "$link_file" + echo "Found $(wc -l < "$link_file") unique trusted links:" + cat "$link_file" + + - name: Check project links + shell: bash + env: + LINK_FILE: ${{ steps.get_project_links.outputs.link-file }} + DOCKER_IMAGE: ${{ inputs.docker-image }} + run: | + set -u + + if ! command -v docker >/dev/null 2>&1; then + echo "::error title=Check Project Links Error::docker is not available. Install Docker or add a pre-step that installs it before using this action." + exit 1 fi - # Explicitly drop localhost and any IP-based hostname before validation. - : > "${link_file}.filtered" - while IFS= read -r url; do - [ -n "$url" ] || continue - - host="${url#*://}" - host="${host%%/*}" - host="${host%%:*}" - host="${host,,}" - host="${host%.}" - - if printf '%s\n' "$host" | grep -Eq '^(localhost|.*\.localhost)$'; then - continue - fi - if printf '%s\n' "$host" | grep -Eq '^[0-9]+(\.[0-9]+){3}$'; then - continue - fi - if printf '%s\n' "$host" | grep -Eq '^(::1|[0-9a-fA-F:]+)$'; then - continue - fi - if printf '%s\n' "$host" | grep -Eq '^(127\.|0\.0\.0\.0|169\.254\.|metadata(\.google\.internal)?)$'; then - continue - fi - if printf '%s\n' "$host" | grep -Eq '^(10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[0-1])\.|fc[0-9a-fA-F:]+|fe[89ab][0-9a-fA-F:]+)$'; then - continue - fi - - printf '%s\n' "$url" >> "${link_file}.filtered" - done < "$link_file" - - if [ -s "${link_file}.filtered" ]; then - mv "${link_file}.filtered" "$link_file" - else - : > "$link_file" + if [ -z "${LINK_FILE:-}" ] || [ ! -f "$LINK_FILE" ]; then + echo "::error title=Check Project Links Error::No project links file was generated." + exit 1 fi - if [ ! -s "$link_file" ]; then - echo "No trusted project links found under $PROJECT_DIRECTORY." + if [ ! -s "$LINK_FILE" ]; then + echo "No trusted project links found." exit 0 fi - echo "Found $(wc -l < "$link_file") unique trusted links:" - cat "$link_file" + echo "Found $(wc -l < "$LINK_FILE") unique trusted links:" + cat "$LINK_FILE" docker run --rm \ - -v "$link_file:/tmp/project_links.txt:ro" \ + -v "$LINK_FILE:/tmp/project_links.txt:ro" \ "$DOCKER_IMAGE" \ sh -ec ' failed=0 diff --git a/check-project-links/find_project_links.py b/check-project-links/find_project_links.py new file mode 100644 index 0000000..074a991 --- /dev/null +++ b/check-project-links/find_project_links.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 + +import argparse +import ipaddress +import os +import sys +from urllib.parse import urlsplit + +try: + import tomllib +except ModuleNotFoundError: # pragma: no cover + raise Exception('tomllib is not available. Please use Python 3.11 or later.') + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description='Collect trusted project URLs from pyproject.toml files.') + parser.add_argument('project_directory', help='Directory containing project pyproject.toml files.') + parser.add_argument('allowed_domains', help='Comma-separated list of allowed domains.') + parser.add_argument('output_path', help='Path to write discovered URLs.') + return parser.parse_args() + + +def is_allowed(hostname: str, allowed: list[str]) -> bool: + if not hostname: + return False + + host = hostname.lower().rstrip('.') + if host == 'localhost' or host.endswith('.localhost'): + return False + + try: + ip = ipaddress.ip_address(host) + except ValueError: + ip = None + + if ip is not None: + return False + + for domain in allowed: + if domain.startswith('*.'): + suffix = domain[2:] + if host == suffix or host.endswith(f'.{suffix}'): + return True + else: + if host == domain: + return True + return False + + +def walk_metadata(value, results, allowed): + if isinstance(value, dict): + for item in value.values(): + walk_metadata(item, results, allowed) + elif isinstance(value, list): + for item in value: + walk_metadata(item, results, allowed) + elif isinstance(value, str): + if not value.startswith('https://'): + return + host = urlsplit(value).hostname + if host and is_allowed(host, allowed): + results.add(value) + + +def main() -> int: + args = parse_args() + project_directory = args.project_directory + allowed_domains = args.allowed_domains + output_path = args.output_path + + allowed = {} + for raw_domain in allowed_domains.split(','): + domain = raw_domain.strip().lower().rstrip('.') + if domain: + allowed.add(domain) + + results: set[str] = set() + + for root, _, files in os.walk(project_directory): + for file_name in files: + if file_name != 'pyproject.toml': + continue + + manifest_path = os.path.join(root, file_name) + try: + with open(manifest_path, 'rb') as manifest_file: + metadata = tomllib.load(manifest_file) + except (OSError, tomllib.TOMLDecodeError): + print(f"Warning: Failed to read or parse pyproject.toml at {manifest_path}", file=sys.stderr) + continue + + walk_metadata(metadata, results, allowed) + + with open(output_path, 'w', encoding='utf-8') as output_file: + for url in sorted(results): + output_file.write(f'{url}\n') + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) From a772e336d4c48a75012600eaa3b111dfc373ce72 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Wed, 2 Sep 2026 17:18:12 -0500 Subject: [PATCH 16/36] update doc --- check-project-links/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/check-project-links/README.md b/check-project-links/README.md index 8544d59..bcb4a67 100644 --- a/check-project-links/README.md +++ b/check-project-links/README.md @@ -44,9 +44,8 @@ steps: ## Behavior -- Uses `rg` to locate `pyproject.toml` files beneath the configured project directory when available. -- Falls back to `find` (discovery) and `grep` (extraction) with a warning if `rg` is not installed; consider adding a pre-step to install `rg` for faster runtime. -- Extracts every `https://...` URL that ends at the first whitespace or double-quote character. +- Uses Python directory walk and [`tomllib`](https://docs.python.org/3/library/tomllib.html) + string search to identify all links in pyproject.toml files under the provided path. +- Extracts strings that are parsable URLs, and URLs from comments - Drops any URL whose hostname is `localhost`, a local loopback address, or any literal IP address before validation. - Deduplicates the list of URLs and writes them to a temporary file. - Validates each URL with the configured Docker image. From 1ef1d05b52e55c8c90bcf4f6fb29af299b8d791b Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Wed, 2 Sep 2026 17:19:09 -0500 Subject: [PATCH 17/36] switch to python impl --- check-project-links/action.yml | 43 +++++-------- check-project-links/find_project_links.py | 75 ++++++++++++++++++----- 2 files changed, 75 insertions(+), 43 deletions(-) diff --git a/check-project-links/action.yml b/check-project-links/action.yml index caaa8de..f6ac900 100644 --- a/check-project-links/action.yml +++ b/check-project-links/action.yml @@ -15,19 +15,25 @@ inputs: runs: using: composite steps: - - name: Get project links - id: get_project_links + - name: Check project links + id: check-project-links shell: bash env: PROJECT_DIRECTORY: ${{ inputs.project-directory }} ALLOWED_DOMAINS: ${{ inputs.allowed-domains }} + DOCKER_IMAGE: ${{ inputs.docker-image }} run: | - set -u + set -euo pipefail if [ ! -d "$PROJECT_DIRECTORY" ]; then echo "::error title=Check Project Links Error::Project directory '$PROJECT_DIRECTORY' does not exist." exit 1 fi + + if ! command -v docker >/dev/null 2>&1; then + echo "::error title=Check Project Links Error::docker is not available. Install Docker or add a pre-step that installs it before using this action." + exit 1 + fi link_file="$(mktemp)" cleanup() { @@ -37,8 +43,6 @@ runs: python3 "$GITHUB_ACTION_PATH/find_project_links.py" "$PROJECT_DIRECTORY" "$ALLOWED_DOMAINS" "$link_file" - echo "link-file=$link_file" >> "$GITHUB_OUTPUT" - if [ ! -s "$link_file" ]; then echo "No trusted project links found under $PROJECT_DIRECTORY." exit 0 @@ -47,34 +51,17 @@ runs: echo "Found $(wc -l < "$link_file") unique trusted links:" cat "$link_file" - - name: Check project links - shell: bash - env: - LINK_FILE: ${{ steps.get_project_links.outputs.link-file }} - DOCKER_IMAGE: ${{ inputs.docker-image }} - run: | - set -u - - if ! command -v docker >/dev/null 2>&1; then - echo "::error title=Check Project Links Error::docker is not available. Install Docker or add a pre-step that installs it before using this action." - exit 1 - fi - - if [ -z "${LINK_FILE:-}" ] || [ ! -f "$LINK_FILE" ]; then - echo "::error title=Check Project Links Error::No project links file was generated." - exit 1 - fi - - if [ ! -s "$LINK_FILE" ]; then + if [ ! -s "$link_file" ]; then echo "No trusted project links found." exit 0 fi - echo "Found $(wc -l < "$LINK_FILE") unique trusted links:" - cat "$LINK_FILE" + echo "Found $(wc -l < "$link_file") unique trusted links:" + cat "$link_file" - docker run --rm \ - -v "$LINK_FILE:/tmp/project_links.txt:ro" \ + unprivileged_user=100:100 # this matches the default, but we want to be explicit about it + docker run -u "${unprivileged_user}" --rm \ + -v "$link_file:/tmp/project_links.txt:ro" \ "$DOCKER_IMAGE" \ sh -ec ' failed=0 diff --git a/check-project-links/find_project_links.py b/check-project-links/find_project_links.py index 074a991..c969801 100644 --- a/check-project-links/find_project_links.py +++ b/check-project-links/find_project_links.py @@ -20,7 +20,19 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -def is_allowed(hostname: str, allowed: list[str]) -> bool: +def is_allowed(hostname: str, allowed: set[str]) -> bool: + """ + Check if the given hostname is allowed based on the provided set of allowed domains. + + >>> is_allowed('example.com', {'example.com'}) + True + + >>> is_allowed('sub.example.com', {'example.com'}) + False + + >>> is_allowed('sub.example.com', {'*.example.com'}) + True + """ if not hostname: return False @@ -36,18 +48,30 @@ def is_allowed(hostname: str, allowed: list[str]) -> bool: if ip is not None: return False - for domain in allowed: - if domain.startswith('*.'): - suffix = domain[2:] - if host == suffix or host.endswith(f'.{suffix}'): - return True - else: - if host == domain: - return True + if host in allowed: + return True + + if any(host.endswith(f'.{domain.lstrip(".*")}') for domain in allowed if domain.startswith('*.')): + return True + return False -def walk_metadata(value, results, allowed): +def safe_under(base_dir: str, candidate_path: str) -> str: + """Return a canonical path that stays under base_dir, or raise ValueError.""" + safe_base = os.path.realpath(base_dir) + safe_candidate = os.path.realpath(candidate_path) + + try: + if os.path.commonpath([safe_base, safe_candidate]) != safe_base: + raise ValueError(f"Path escapes base directory: {candidate_path!r}") + except ValueError as exc: + raise ValueError(f"Path escapes base directory: {candidate_path!r}") from exc + + return safe_candidate + + +def walk_metadata(value, results: set[str], allowed: set[str]) -> None: if isinstance(value, dict): for item in value.values(): walk_metadata(item, results, allowed) @@ -65,10 +89,12 @@ def walk_metadata(value, results, allowed): def main() -> int: args = parse_args() project_directory = args.project_directory + safe_project_directory = os.path.realpath(project_directory, strict=True) + safe_project_directory = safe_under(os.getcwd(), safe_project_directory) allowed_domains = args.allowed_domains - output_path = args.output_path + safe_output_path = safe_under("/tmp", os.path.realpath(args.output_path, strict=True)) - allowed = {} + allowed: set[str] = set() for raw_domain in allowed_domains.split(','): domain = raw_domain.strip().lower().rstrip('.') if domain: @@ -76,12 +102,12 @@ def main() -> int: results: set[str] = set() - for root, _, files in os.walk(project_directory): + for root, _, files in os.walk(safe_project_directory): for file_name in files: if file_name != 'pyproject.toml': continue - manifest_path = os.path.join(root, file_name) + manifest_path = safe_under(safe_project_directory, os.path.join(root, file_name)) try: with open(manifest_path, 'rb') as manifest_file: metadata = tomllib.load(manifest_file) @@ -90,8 +116,27 @@ def main() -> int: continue walk_metadata(metadata, results, allowed) + # Also find any commented URLs in the pyproject.toml file + try: + with open(manifest_path, 'r', encoding='utf-8') as manifest_file: + for line in manifest_file: + line = line.strip() + prefix, comment = line.split('#', maxsplit=1) if '#' in line else (line, '') + if comment.strip(): + comment = comment.strip() + if comment.startswith('https://'): + host = urlsplit(comment).hostname # handles extra at the end just fine + if host and is_allowed(host, allowed): + results.add(comment) + except OSError: + print(f"Warning: Failed to read pyproject.toml at {manifest_path}", file=sys.stderr) + continue + + output_directory = os.path.dirname(safe_output_path) + if output_directory: + os.makedirs(output_directory, exist_ok=True) - with open(output_path, 'w', encoding='utf-8') as output_file: + with open(safe_output_path, 'w', encoding='utf-8') as output_file: for url in sorted(results): output_file.write(f'{url}\n') From 4c195441daff03b1d053cf1895c0f4c0093fa7ea Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Wed, 2 Sep 2026 17:28:12 -0500 Subject: [PATCH 18/36] all private --- check-project-links/find_project_links.py | 36 +++++++++++------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/check-project-links/find_project_links.py b/check-project-links/find_project_links.py index c969801..1b1203c 100644 --- a/check-project-links/find_project_links.py +++ b/check-project-links/find_project_links.py @@ -12,7 +12,7 @@ raise Exception('tomllib is not available. Please use Python 3.11 or later.') -def parse_args() -> argparse.Namespace: +def _parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description='Collect trusted project URLs from pyproject.toml files.') parser.add_argument('project_directory', help='Directory containing project pyproject.toml files.') parser.add_argument('allowed_domains', help='Comma-separated list of allowed domains.') @@ -20,17 +20,17 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -def is_allowed(hostname: str, allowed: set[str]) -> bool: +def _is_allowed(hostname: str, allowed: set[str]) -> bool: """ Check if the given hostname is allowed based on the provided set of allowed domains. - >>> is_allowed('example.com', {'example.com'}) + >>> _is_allowed('example.com', {'example.com'}) True - >>> is_allowed('sub.example.com', {'example.com'}) + >>> _is_allowed('sub.example.com', {'example.com'}) False - >>> is_allowed('sub.example.com', {'*.example.com'}) + >>> _is_allowed('sub.example.com', {'*.example.com'}) True """ if not hostname: @@ -57,7 +57,7 @@ def is_allowed(hostname: str, allowed: set[str]) -> bool: return False -def safe_under(base_dir: str, candidate_path: str) -> str: +def _safe_under(base_dir: str, candidate_path: str) -> str: """Return a canonical path that stays under base_dir, or raise ValueError.""" safe_base = os.path.realpath(base_dir) safe_candidate = os.path.realpath(candidate_path) @@ -71,28 +71,28 @@ def safe_under(base_dir: str, candidate_path: str) -> str: return safe_candidate -def walk_metadata(value, results: set[str], allowed: set[str]) -> None: +def _walk_metadata(value, results: set[str], allowed: set[str]) -> None: if isinstance(value, dict): for item in value.values(): - walk_metadata(item, results, allowed) + _walk_metadata(item, results, allowed) elif isinstance(value, list): for item in value: - walk_metadata(item, results, allowed) + _walk_metadata(item, results, allowed) elif isinstance(value, str): if not value.startswith('https://'): return host = urlsplit(value).hostname - if host and is_allowed(host, allowed): + if host and _is_allowed(host, allowed): results.add(value) -def main() -> int: - args = parse_args() +def _main() -> int: + args = _parse_args() project_directory = args.project_directory safe_project_directory = os.path.realpath(project_directory, strict=True) - safe_project_directory = safe_under(os.getcwd(), safe_project_directory) + safe_project_directory = _safe_under(os.getcwd(), safe_project_directory) allowed_domains = args.allowed_domains - safe_output_path = safe_under("/tmp", os.path.realpath(args.output_path, strict=True)) + safe_output_path = _safe_under("/tmp", os.path.realpath(args.output_path, strict=True)) allowed: set[str] = set() for raw_domain in allowed_domains.split(','): @@ -107,7 +107,7 @@ def main() -> int: if file_name != 'pyproject.toml': continue - manifest_path = safe_under(safe_project_directory, os.path.join(root, file_name)) + manifest_path = _safe_under(safe_project_directory, os.path.join(root, file_name)) try: with open(manifest_path, 'rb') as manifest_file: metadata = tomllib.load(manifest_file) @@ -115,7 +115,7 @@ def main() -> int: print(f"Warning: Failed to read or parse pyproject.toml at {manifest_path}", file=sys.stderr) continue - walk_metadata(metadata, results, allowed) + _walk_metadata(metadata, results, allowed) # Also find any commented URLs in the pyproject.toml file try: with open(manifest_path, 'r', encoding='utf-8') as manifest_file: @@ -126,7 +126,7 @@ def main() -> int: comment = comment.strip() if comment.startswith('https://'): host = urlsplit(comment).hostname # handles extra at the end just fine - if host and is_allowed(host, allowed): + if host and _is_allowed(host, allowed): results.add(comment) except OSError: print(f"Warning: Failed to read pyproject.toml at {manifest_path}", file=sys.stderr) @@ -144,4 +144,4 @@ def main() -> int: if __name__ == '__main__': - sys.exit(main()) + sys.exit(_main()) From d9ead733804df8c5ea6f001903616c9a8bc2854c Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Wed, 2 Sep 2026 17:28:37 -0500 Subject: [PATCH 19/36] fix docstring --- check-project-links/find_project_links.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/check-project-links/find_project_links.py b/check-project-links/find_project_links.py index 1b1203c..e8655b0 100644 --- a/check-project-links/find_project_links.py +++ b/check-project-links/find_project_links.py @@ -21,8 +21,7 @@ def _parse_args() -> argparse.Namespace: def _is_allowed(hostname: str, allowed: set[str]) -> bool: - """ - Check if the given hostname is allowed based on the provided set of allowed domains. + """Check if the given hostname is allowed based on the provided set of allowed domains. >>> _is_allowed('example.com', {'example.com'}) True From d37c307d91a51a95937bd5159469b4ef11b131f4 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Wed, 2 Sep 2026 17:29:38 -0500 Subject: [PATCH 20/36] format --- check-project-links/find_project_links.py | 53 ++++++++++++++--------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/check-project-links/find_project_links.py b/check-project-links/find_project_links.py index e8655b0..0d9c5ce 100644 --- a/check-project-links/find_project_links.py +++ b/check-project-links/find_project_links.py @@ -9,14 +9,18 @@ try: import tomllib except ModuleNotFoundError: # pragma: no cover - raise Exception('tomllib is not available. Please use Python 3.11 or later.') + raise Exception("tomllib is not available. Please use Python 3.11 or later.") def _parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description='Collect trusted project URLs from pyproject.toml files.') - parser.add_argument('project_directory', help='Directory containing project pyproject.toml files.') - parser.add_argument('allowed_domains', help='Comma-separated list of allowed domains.') - parser.add_argument('output_path', help='Path to write discovered URLs.') + parser = argparse.ArgumentParser( + description="Collect trusted project URLs from pyproject.toml files." + ) + parser.add_argument( + "project_directory", help="Directory containing project pyproject.toml files." + ) + parser.add_argument("allowed_domains", help="Comma-separated list of allowed domains.") + parser.add_argument("output_path", help="Path to write discovered URLs.") return parser.parse_args() @@ -35,8 +39,8 @@ def _is_allowed(hostname: str, allowed: set[str]) -> bool: if not hostname: return False - host = hostname.lower().rstrip('.') - if host == 'localhost' or host.endswith('.localhost'): + host = hostname.lower().rstrip(".") + if host == "localhost" or host.endswith(".localhost"): return False try: @@ -50,7 +54,9 @@ def _is_allowed(hostname: str, allowed: set[str]) -> bool: if host in allowed: return True - if any(host.endswith(f'.{domain.lstrip(".*")}') for domain in allowed if domain.startswith('*.')): + if any( + host.endswith(f'.{domain.lstrip(".*")}') for domain in allowed if domain.startswith("*.") + ): return True return False @@ -78,7 +84,7 @@ def _walk_metadata(value, results: set[str], allowed: set[str]) -> None: for item in value: _walk_metadata(item, results, allowed) elif isinstance(value, str): - if not value.startswith('https://'): + if not value.startswith("https://"): return host = urlsplit(value).hostname if host and _is_allowed(host, allowed): @@ -94,8 +100,8 @@ def _main() -> int: safe_output_path = _safe_under("/tmp", os.path.realpath(args.output_path, strict=True)) allowed: set[str] = set() - for raw_domain in allowed_domains.split(','): - domain = raw_domain.strip().lower().rstrip('.') + for raw_domain in allowed_domains.split(","): + domain = raw_domain.strip().lower().rstrip(".") if domain: allowed.add(domain) @@ -103,28 +109,33 @@ def _main() -> int: for root, _, files in os.walk(safe_project_directory): for file_name in files: - if file_name != 'pyproject.toml': + if file_name != "pyproject.toml": continue manifest_path = _safe_under(safe_project_directory, os.path.join(root, file_name)) try: - with open(manifest_path, 'rb') as manifest_file: + with open(manifest_path, "rb") as manifest_file: metadata = tomllib.load(manifest_file) except (OSError, tomllib.TOMLDecodeError): - print(f"Warning: Failed to read or parse pyproject.toml at {manifest_path}", file=sys.stderr) + print( + f"Warning: Failed to read or parse pyproject.toml at {manifest_path}", + file=sys.stderr, + ) continue _walk_metadata(metadata, results, allowed) # Also find any commented URLs in the pyproject.toml file try: - with open(manifest_path, 'r', encoding='utf-8') as manifest_file: + with open(manifest_path, "r", encoding="utf-8") as manifest_file: for line in manifest_file: line = line.strip() - prefix, comment = line.split('#', maxsplit=1) if '#' in line else (line, '') + prefix, comment = line.split("#", maxsplit=1) if "#" in line else (line, "") if comment.strip(): comment = comment.strip() - if comment.startswith('https://'): - host = urlsplit(comment).hostname # handles extra at the end just fine + if comment.startswith("https://"): + host = urlsplit( + comment + ).hostname # handles extra at the end just fine if host and _is_allowed(host, allowed): results.add(comment) except OSError: @@ -135,12 +146,12 @@ def _main() -> int: if output_directory: os.makedirs(output_directory, exist_ok=True) - with open(safe_output_path, 'w', encoding='utf-8') as output_file: + with open(safe_output_path, "w", encoding="utf-8") as output_file: for url in sorted(results): - output_file.write(f'{url}\n') + output_file.write(f"{url}\n") return 0 -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(_main()) From ad3ece36623aebbd0047633e528d690738b15565 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Wed, 2 Sep 2026 17:41:48 -0500 Subject: [PATCH 21/36] also not a public module --- .../{find_project_links.py => _find_project_links.py} | 0 check-project-links/action.yml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename check-project-links/{find_project_links.py => _find_project_links.py} (100%) diff --git a/check-project-links/find_project_links.py b/check-project-links/_find_project_links.py similarity index 100% rename from check-project-links/find_project_links.py rename to check-project-links/_find_project_links.py diff --git a/check-project-links/action.yml b/check-project-links/action.yml index f6ac900..46593dc 100644 --- a/check-project-links/action.yml +++ b/check-project-links/action.yml @@ -41,7 +41,7 @@ runs: } trap cleanup EXIT - python3 "$GITHUB_ACTION_PATH/find_project_links.py" "$PROJECT_DIRECTORY" "$ALLOWED_DOMAINS" "$link_file" + python3 "$GITHUB_ACTION_PATH/_find_project_links.py" "$PROJECT_DIRECTORY" "$ALLOWED_DOMAINS" "$link_file" if [ ! -s "$link_file" ]; then echo "No trusted project links found under $PROJECT_DIRECTORY." From bcbd82feaf83e3556022616ec856043ca28cf9df Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Wed, 2 Sep 2026 17:45:23 -0500 Subject: [PATCH 22/36] set the file as world read so the docker user can read it --- check-project-links/action.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/check-project-links/action.yml b/check-project-links/action.yml index 46593dc..71aef70 100644 --- a/check-project-links/action.yml +++ b/check-project-links/action.yml @@ -40,8 +40,10 @@ runs: rm -f "$link_file" } trap cleanup EXIT + chmod 0644 "$link_file" python3 "$GITHUB_ACTION_PATH/_find_project_links.py" "$PROJECT_DIRECTORY" "$ALLOWED_DOMAINS" "$link_file" + chmod 0644 "$link_file" if [ ! -s "$link_file" ]; then echo "No trusted project links found under $PROJECT_DIRECTORY." From 970e7ad01de5a98796ce4bb63a8f3180475be0f8 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 3 Sep 2026 10:12:54 -0500 Subject: [PATCH 23/36] extract comment searching to method --- check-project-links/_find_project_links.py | 35 +++++++++++----------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/check-project-links/_find_project_links.py b/check-project-links/_find_project_links.py index 0d9c5ce..a4e7c89 100644 --- a/check-project-links/_find_project_links.py +++ b/check-project-links/_find_project_links.py @@ -91,6 +91,23 @@ def _walk_metadata(value, results: set[str], allowed: set[str]) -> None: results.add(value) +def _find_commented_links(manifest_path: str, results: set[str], allowed: set[str]) -> None: + """Find URLs embedded in comments within a pyproject.toml file.""" + try: + with open(manifest_path, "r", encoding="utf-8") as manifest_file: + for line in manifest_file: + line = line.strip() + prefix, comment = line.split("#", maxsplit=1) if "#" in line else (line, "") + if comment.strip(): + comment = comment.strip() + if comment.startswith("https://"): + host = urlsplit(comment).hostname # handles extra at the end just fine + if host and _is_allowed(host, allowed): + results.add(comment) + except OSError: + print(f"Warning: Failed to read pyproject.toml at {manifest_path}", file=sys.stderr) + + def _main() -> int: args = _parse_args() project_directory = args.project_directory @@ -124,23 +141,7 @@ def _main() -> int: continue _walk_metadata(metadata, results, allowed) - # Also find any commented URLs in the pyproject.toml file - try: - with open(manifest_path, "r", encoding="utf-8") as manifest_file: - for line in manifest_file: - line = line.strip() - prefix, comment = line.split("#", maxsplit=1) if "#" in line else (line, "") - if comment.strip(): - comment = comment.strip() - if comment.startswith("https://"): - host = urlsplit( - comment - ).hostname # handles extra at the end just fine - if host and _is_allowed(host, allowed): - results.add(comment) - except OSError: - print(f"Warning: Failed to read pyproject.toml at {manifest_path}", file=sys.stderr) - continue + _find_commented_links(manifest_path, results, allowed) output_directory = os.path.dirname(safe_output_path) if output_directory: From 97dbe0114e0a2334a5d58084932cbbead38c3f9b Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 3 Sep 2026 10:13:30 -0500 Subject: [PATCH 24/36] remove duplicate check --- check-project-links/action.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/check-project-links/action.yml b/check-project-links/action.yml index 71aef70..16a2ca8 100644 --- a/check-project-links/action.yml +++ b/check-project-links/action.yml @@ -53,14 +53,6 @@ runs: echo "Found $(wc -l < "$link_file") unique trusted links:" cat "$link_file" - if [ ! -s "$link_file" ]; then - echo "No trusted project links found." - exit 0 - fi - - echo "Found $(wc -l < "$link_file") unique trusted links:" - cat "$link_file" - unprivileged_user=100:100 # this matches the default, but we want to be explicit about it docker run -u "${unprivileged_user}" --rm \ -v "$link_file:/tmp/project_links.txt:ro" \ From 9ecc8aa5bb74b50836a8a1947291b6e3bd462eab Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 3 Sep 2026 10:14:49 -0500 Subject: [PATCH 25/36] use RUNNER_TEMP --- check-project-links/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check-project-links/action.yml b/check-project-links/action.yml index 16a2ca8..3768b4d 100644 --- a/check-project-links/action.yml +++ b/check-project-links/action.yml @@ -35,7 +35,7 @@ runs: exit 1 fi - link_file="$(mktemp)" + link_file="$(mktemp -p \"$RUNNER_TEMP\")" cleanup() { rm -f "$link_file" } From ee8f981b79c5f23fbbb517f3eae78e19b2631a6b Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 3 Sep 2026 10:21:33 -0500 Subject: [PATCH 26/36] use RUNNER_TEMP and test not changing mode --- check-project-links/_find_project_links.py | 6 +++++- check-project-links/action.yml | 4 +--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/check-project-links/_find_project_links.py b/check-project-links/_find_project_links.py index a4e7c89..fa2b429 100644 --- a/check-project-links/_find_project_links.py +++ b/check-project-links/_find_project_links.py @@ -114,7 +114,11 @@ def _main() -> int: safe_project_directory = os.path.realpath(project_directory, strict=True) safe_project_directory = _safe_under(os.getcwd(), safe_project_directory) allowed_domains = args.allowed_domains - safe_output_path = _safe_under("/tmp", os.path.realpath(args.output_path, strict=True)) + + runner_temp = os.environ.get("RUNNER_TEMP") + if not runner_temp: + raise RuntimeError("RUNNER_TEMP is required for output files.") + safe_output_path = _safe_under(runner_temp, os.path.realpath(args.output_path, strict=True)) allowed: set[str] = set() for raw_domain in allowed_domains.split(","): diff --git a/check-project-links/action.yml b/check-project-links/action.yml index 3768b4d..cceae6c 100644 --- a/check-project-links/action.yml +++ b/check-project-links/action.yml @@ -35,15 +35,13 @@ runs: exit 1 fi - link_file="$(mktemp -p \"$RUNNER_TEMP\")" + link_file="$(mktemp -p "$RUNNER_TEMP")" cleanup() { rm -f "$link_file" } trap cleanup EXIT - chmod 0644 "$link_file" python3 "$GITHUB_ACTION_PATH/_find_project_links.py" "$PROJECT_DIRECTORY" "$ALLOWED_DOMAINS" "$link_file" - chmod 0644 "$link_file" if [ ! -s "$link_file" ]; then echo "No trusted project links found under $PROJECT_DIRECTORY." From 2da41f96a6bf037791846d1d4c42cf961831c7a9 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 3 Sep 2026 10:28:03 -0500 Subject: [PATCH 27/36] turns out chmod is required because mktemp makes it too restricted --- check-project-links/action.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/check-project-links/action.yml b/check-project-links/action.yml index cceae6c..5a78e4f 100644 --- a/check-project-links/action.yml +++ b/check-project-links/action.yml @@ -41,6 +41,8 @@ runs: } trap cleanup EXIT + chmod 644 "$link_file" + python3 "$GITHUB_ACTION_PATH/_find_project_links.py" "$PROJECT_DIRECTORY" "$ALLOWED_DOMAINS" "$link_file" if [ ! -s "$link_file" ]; then From d69e2324f44dfa553ed3c22efcb6e7acf30611ac Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 3 Sep 2026 10:34:51 -0500 Subject: [PATCH 28/36] don't require output to be in temp, just resolve it --- check-project-links/_find_project_links.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/check-project-links/_find_project_links.py b/check-project-links/_find_project_links.py index fa2b429..177240f 100644 --- a/check-project-links/_find_project_links.py +++ b/check-project-links/_find_project_links.py @@ -114,11 +114,7 @@ def _main() -> int: safe_project_directory = os.path.realpath(project_directory, strict=True) safe_project_directory = _safe_under(os.getcwd(), safe_project_directory) allowed_domains = args.allowed_domains - - runner_temp = os.environ.get("RUNNER_TEMP") - if not runner_temp: - raise RuntimeError("RUNNER_TEMP is required for output files.") - safe_output_path = _safe_under(runner_temp, os.path.realpath(args.output_path, strict=True)) + safe_output_path = os.path.realpath(args.output_path, strict=True) allowed: set[str] = set() for raw_domain in allowed_domains.split(","): From a24c0d843b627a5e147b67d09226dbae85f5688e Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 3 Sep 2026 10:42:15 -0500 Subject: [PATCH 29/36] add type annotation --- check-project-links/_find_project_links.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/check-project-links/_find_project_links.py b/check-project-links/_find_project_links.py index 177240f..b9085cd 100644 --- a/check-project-links/_find_project_links.py +++ b/check-project-links/_find_project_links.py @@ -4,6 +4,7 @@ import ipaddress import os import sys +import typing from urllib.parse import urlsplit try: @@ -76,7 +77,9 @@ def _safe_under(base_dir: str, candidate_path: str) -> str: return safe_candidate -def _walk_metadata(value, results: set[str], allowed: set[str]) -> None: +def _walk_metadata( + value: typing.Union[dict, list, str], results: set[str], allowed: set[str] +) -> None: if isinstance(value, dict): for item in value.values(): _walk_metadata(item, results, allowed) From e7461a426671c6613c4bdd48423c95870331c3c2 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 3 Sep 2026 10:44:12 -0500 Subject: [PATCH 30/36] just say any --- check-project-links/_find_project_links.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/check-project-links/_find_project_links.py b/check-project-links/_find_project_links.py index b9085cd..cd6584e 100644 --- a/check-project-links/_find_project_links.py +++ b/check-project-links/_find_project_links.py @@ -77,9 +77,7 @@ def _safe_under(base_dir: str, candidate_path: str) -> str: return safe_candidate -def _walk_metadata( - value: typing.Union[dict, list, str], results: set[str], allowed: set[str] -) -> None: +def _walk_metadata(value: typing.Any, results: set[str], allowed: set[str]) -> None: if isinstance(value, dict): for item in value.values(): _walk_metadata(item, results, allowed) From c77494fd15aea2c27175a038c6f572c5a0ab9c75 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 3 Sep 2026 10:50:50 -0500 Subject: [PATCH 31/36] let mypy pass on 3.10 --- check-project-links/_find_project_links.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check-project-links/_find_project_links.py b/check-project-links/_find_project_links.py index cd6584e..29d3244 100644 --- a/check-project-links/_find_project_links.py +++ b/check-project-links/_find_project_links.py @@ -8,7 +8,7 @@ from urllib.parse import urlsplit try: - import tomllib + import tomllib # type: ignore[import-not-found] except ModuleNotFoundError: # pragma: no cover raise Exception("tomllib is not available. Please use Python 3.11 or later.") From ced8a580d68c10ac2c7fce76b1678eab3fc3185d Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 3 Sep 2026 11:31:43 -0500 Subject: [PATCH 32/36] switch to just regex parsing --- check-project-links/_find_project_links.py | 60 +++++++--------------- 1 file changed, 19 insertions(+), 41 deletions(-) diff --git a/check-project-links/_find_project_links.py b/check-project-links/_find_project_links.py index 29d3244..8b4e4a1 100644 --- a/check-project-links/_find_project_links.py +++ b/check-project-links/_find_project_links.py @@ -3,15 +3,10 @@ import argparse import ipaddress import os +import re import sys -import typing from urllib.parse import urlsplit -try: - import tomllib # type: ignore[import-not-found] -except ModuleNotFoundError: # pragma: no cover - raise Exception("tomllib is not available. Please use Python 3.11 or later.") - def _parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( @@ -77,34 +72,28 @@ def _safe_under(base_dir: str, candidate_path: str) -> str: return safe_candidate -def _walk_metadata(value: typing.Any, results: set[str], allowed: set[str]) -> None: - if isinstance(value, dict): - for item in value.values(): - _walk_metadata(item, results, allowed) - elif isinstance(value, list): - for item in value: - _walk_metadata(item, results, allowed) - elif isinstance(value, str): - if not value.startswith("https://"): - return - host = urlsplit(value).hostname - if host and _is_allowed(host, allowed): - results.add(value) +_URL_RE = re.compile(r'(https://.+?)(?:"|\s)') + + +def _extract_links_from_line(line: str) -> list[str]: + """Extract URLs from a single line, stopping before a quote or whitespace.""" + matches: list[str] = [] + for match in re.finditer(_URL_RE, line): + url = match.group(1) + if url: + matches.append(url) + return matches -def _find_commented_links(manifest_path: str, results: set[str], allowed: set[str]) -> None: - """Find URLs embedded in comments within a pyproject.toml file.""" +def _find_project_links(manifest_path: str, results: set[str], allowed: set[str]) -> None: + """Find URLs within a pyproject.toml file without parsing TOML.""" try: with open(manifest_path, "r", encoding="utf-8") as manifest_file: for line in manifest_file: - line = line.strip() - prefix, comment = line.split("#", maxsplit=1) if "#" in line else (line, "") - if comment.strip(): - comment = comment.strip() - if comment.startswith("https://"): - host = urlsplit(comment).hostname # handles extra at the end just fine - if host and _is_allowed(host, allowed): - results.add(comment) + for url in _extract_links_from_line(line): + host = urlsplit(url).hostname + if host and _is_allowed(host, allowed): + results.add(url) except OSError: print(f"Warning: Failed to read pyproject.toml at {manifest_path}", file=sys.stderr) @@ -131,18 +120,7 @@ def _main() -> int: continue manifest_path = _safe_under(safe_project_directory, os.path.join(root, file_name)) - try: - with open(manifest_path, "rb") as manifest_file: - metadata = tomllib.load(manifest_file) - except (OSError, tomllib.TOMLDecodeError): - print( - f"Warning: Failed to read or parse pyproject.toml at {manifest_path}", - file=sys.stderr, - ) - continue - - _walk_metadata(metadata, results, allowed) - _find_commented_links(manifest_path, results, allowed) + _find_project_links(manifest_path, results, allowed) output_directory = os.path.dirname(safe_output_path) if output_directory: From 99c7d0ba5b85fdd7a419dd9a124d0901d206b35a Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 3 Sep 2026 11:39:27 -0500 Subject: [PATCH 33/36] add comment --- check-project-links/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check-project-links/action.yml b/check-project-links/action.yml index 5a78e4f..c342a8f 100644 --- a/check-project-links/action.yml +++ b/check-project-links/action.yml @@ -41,7 +41,7 @@ runs: } trap cleanup EXIT - chmod 644 "$link_file" + chmod 644 "$link_file" # make the file world-readable so that the unprivileged user in the docker container can read it python3 "$GITHUB_ACTION_PATH/_find_project_links.py" "$PROJECT_DIRECTORY" "$ALLOWED_DOMAINS" "$link_file" From e410a1c73b022f8589e0476351e061b199d83ac2 Mon Sep 17 00:00:00 2001 From: mshafer-NI <23644905+mshafer-NI@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:41:25 -0500 Subject: [PATCH 34/36] Apply batched suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- check-project-links/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/check-project-links/README.md b/check-project-links/README.md index bcb4a67..6a03ff7 100644 --- a/check-project-links/README.md +++ b/check-project-links/README.md @@ -36,7 +36,7 @@ steps: - uses: actions/checkout@v0 - name: Check project links - uses: ni/python-actions/check-project-links@v1 + uses: ni/python-actions/check-project-links@v0 with: project-directory: . docker-image: curlimages/curl:8.22.0 @@ -44,8 +44,8 @@ steps: ## Behavior -- Uses Python directory walk and [`tomllib`](https://docs.python.org/3/library/tomllib.html) + string search to identify all links in pyproject.toml files under the provided path. -- Extracts strings that are parsable URLs, and URLs from comments +- Uses Python directory walk + regex search to identify `https://` links in `pyproject.toml` files under the provided path. +- Extracts `https://` substrings (including from comments) and filters them by the configured allowed domains before validation. - Drops any URL whose hostname is `localhost`, a local loopback address, or any literal IP address before validation. - Deduplicates the list of URLs and writes them to a temporary file. - Validates each URL with the configured Docker image. From 4b805eb92e902587b1c74c2698d6f9d406ee1bf4 Mon Sep 17 00:00:00 2001 From: mshafer-NI <23644905+mshafer-NI@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:41:59 -0500 Subject: [PATCH 35/36] Simplify exit handling for project link checks Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- check-project-links/action.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/check-project-links/action.yml b/check-project-links/action.yml index c342a8f..2055a9c 100644 --- a/check-project-links/action.yml +++ b/check-project-links/action.yml @@ -78,9 +78,6 @@ runs: exit "$failed" ' - status=$? - if [ "$status" -ne 0 ]; then - exit 1 - fi + # `docker run` failures already cause this step to fail due to `set -e`. echo "Success: No checked project links returned 4xx responses." From 92c4c6ea66aa5b165b1c7be1a98b97731f6a92c8 Mon Sep 17 00:00:00 2001 From: Matthew Shafer Date: Thu, 3 Sep 2026 12:45:49 -0500 Subject: [PATCH 36/36] tone down the sanitizing --- check-project-links/_find_project_links.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/check-project-links/_find_project_links.py b/check-project-links/_find_project_links.py index 8b4e4a1..5b99398 100644 --- a/check-project-links/_find_project_links.py +++ b/check-project-links/_find_project_links.py @@ -100,11 +100,9 @@ def _find_project_links(manifest_path: str, results: set[str], allowed: set[str] def _main() -> int: args = _parse_args() - project_directory = args.project_directory - safe_project_directory = os.path.realpath(project_directory, strict=True) - safe_project_directory = _safe_under(os.getcwd(), safe_project_directory) + safe_project_directory = _safe_under(os.getcwd(), args.project_directory) allowed_domains = args.allowed_domains - safe_output_path = os.path.realpath(args.output_path, strict=True) + output_path = args.output_path allowed: set[str] = set() for raw_domain in allowed_domains.split(","): @@ -122,11 +120,11 @@ def _main() -> int: manifest_path = _safe_under(safe_project_directory, os.path.join(root, file_name)) _find_project_links(manifest_path, results, allowed) - output_directory = os.path.dirname(safe_output_path) + output_directory = os.path.dirname(output_path) if output_directory: os.makedirs(output_directory, exist_ok=True) - with open(safe_output_path, "w", encoding="utf-8") as output_file: + with open(output_path, "w", encoding="utf-8") as output_file: for url in sorted(results): output_file.write(f"{url}\n")