diff --git a/.github/workflows/refresh-translation-issues.yml b/.github/workflows/refresh-translation-issues.yml new file mode 100644 index 00000000..8458b552 --- /dev/null +++ b/.github/workflows/refresh-translation-issues.yml @@ -0,0 +1,164 @@ +name: Refresh translation issues + +# Updates the GitHub issues used to track the work in the translations. +# Each language has a parent issue with a table with the overall stats, plus +# one sub-issue per catalog (`.po` file). +# +# When `.po` files are merged, the stats are recalculated and the issues are +# rewritten. It only rewrites the body of the issue if there are changes. +# +# It uses labels, titles and a hidden HTML comment in the body to find the +# right issue to update. +# +# It does not open new issues for a new language, this step has to be done +# manually by a maintainer. +# +# In case of issues that need a maintainer to act on, it posts a comment to the issue +# with the Translation Maintenance Log issue (label `po-refresh-tracker`), which +# will appear in the `#translations` Slack channel. +on: + push: + branches: [main] + paths: + - "locales/**/*.po" + workflow_dispatch: + +permissions: + contents: read + issues: write + +# A second push arriving while one is running should cancel the first one. +concurrency: + group: refresh-translation-issues + cancel-in-progress: true + +env: + # Label to look for the Translation Maintenance Log issue (post to `#translations` Slack channel). + TRACKER_LABEL: po-refresh-tracker + +jobs: + refresh: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + # install `babel` to read the catalogs (`.po` files) + - name: Install the catalog reader + run: python -m pip install babel + + - name: Collect the issues that track each language + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const { owner, repo } = context.repo; + + // The locale directories are the list of languages, and `lang-XX` + // is the label convention label-translations.yml uses. + const locales = fs.readdirSync('locales', { withFileTypes: true }) + .filter(entry => entry.isDirectory()) + .map(entry => entry.name) + .sort(); + + const found = {}; + for (const locale of locales) { + const label = `lang-${locale.toUpperCase()}`; + // A label no issue carries comes back empty so a language without issues + // needs no special case. + const issues = await github.paginate(github.rest.issues.listForRepo, { + owner, repo, labels: label, state: 'open', per_page: 100, + }); + // Drops PRs with language labels. + found[locale] = issues + .filter(issue => !issue.pull_request) + .map(issue => ({ + number: issue.number, + title: issue.title, + body: issue.body || '', + })); + core.info( + `${locale}: ${found[locale].length} open issue(s) labelled ${label}.`, + ); + } + fs.writeFileSync('issues.json', JSON.stringify(found)); + + # All checks happen in Python so logic can be tested with pytest. + - name: Work out which issues need rewriting + id: render + run: | + python scripts/translation/refresh_translation_issues.py \ + issues.json bodies.json report.md + if [ -s report.md ]; then + echo "report=true" >> "$GITHUB_OUTPUT" + cat report.md + else + echo "report=false" >> "$GITHUB_OUTPUT" + fi + + - name: Rewrite them + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const { owner, repo } = context.repo; + const updates = JSON.parse(fs.readFileSync('bodies.json', 'utf8')); + + if (updates.length === 0) { + core.info('Every translation issue is already updated.'); + return; + } + for (const { number, body } of updates) { + // Only update the body so maintainers can change labels and titles. + await github.rest.issues.update( + { owner, repo, issue_number: number, body }, + ); + core.info(`Rewrote #${number}.`); + } + await core.summary + .addRaw(`Rewrote ${updates.length} translation issue(s).`) + .write(); + + - name: Report anything that needs a maintainer + if: steps.render.outputs.report == 'true' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const label = process.env.TRACKER_LABEL; + const { owner, repo } = context.repo; + const report = fs.readFileSync('report.md', 'utf8'); + + const found = await github.paginate(github.rest.issues.listForRepo, { + owner, repo, labels: label, state: 'all', per_page: 100, + }); + // This endpoint counts pull requests as issues, so drop those. + const issues = found.filter(i => !i.pull_request); + + if (issues.length === 0) { + // If the Translation Maintenance Log issue is not found (nothing with the expected label) + // create a warning rather than a failure. + core.warning( + `No issue carries the \`${label}\` label, so there is nowhere to ` + + `report this. Create the translation tracking issue and apply that label.` + + `\n\n${report}`, + ); + return; + } + if (issues.length > 1) { + // If there are multiple issues with the translation maintenance label, use the lowest-numbered one and warn. + core.warning( + `${issues.length} issues carry \`${label}\`: ` + + `${issues.map(i => '#' + i.number).join(', ')}. ` + + `Using the lowest-numbered one.`, + ); + } + const issue = issues.sort((a, b) => a.number - b.number)[0]; + + await github.rest.issues.createComment( + { owner, repo, issue_number: issue.number, body: report }, + ); + core.info(`Commented on #${issue.number}.`); diff --git a/noxfile.py b/noxfile.py index 56d1dcbe..075bf3c9 100644 --- a/noxfile.py +++ b/noxfile.py @@ -437,9 +437,11 @@ def test_translation_scripts(session): """ Run the unit tests for the translation helper scripts. - Only pytest is installed since it's the only thing the scripts under test need. + Only pytest and babel are installed: babel because stats.py reads the catalogs + with it. Installing the project would pull in sphinx and plotly to run tests + that never build anything. """ - session.install("pytest") + session.install("pytest", "babel") session.run("pytest", str(TRANSLATION_SCRIPTS_DIR), *session.posargs) diff --git a/pyproject.toml b/pyproject.toml index 5edf2d18..c53e6868 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,9 +4,7 @@ build-backend = "hatchling.build" [project] name = "python-package-guide" -dynamic = [ - "version" -] +dynamic = ["version"] dependencies = [ "pydata-sphinx-theme==0.20.0", "myst-nb", @@ -49,3 +47,4 @@ version.source = "vcs" [tool.codespell] ignore-words = "codespell-ignore.txt" skip = "./.git,./.nox,./_static,./_build,codespell-ignore.txt,*.svg" +ignore-sic = true diff --git a/scripts/translation/child-issue-template.md b/scripts/translation/child-issue-template.md new file mode 100644 index 00000000..f8f940e0 --- /dev/null +++ b/scripts/translation/child-issue-template.md @@ -0,0 +1,34 @@ +> _This issue is created automatically and should not be edited directly._ + +This issue is for translating **[`{{FILENAME}}`]({{FILE_URL}})** into {{LANGUAGE}}. It is one part of the larger effort tracked in the main issue: {{MAIN_ISSUE_URL}}. Start there if you want the full picture and the setup steps. + +Thank you for helping! If this is your first open source contribution, you are in the right place. + +> **`index.po` comes first.** It holds the guide's landing page, and translating it is what lets us publish the {{LANGUAGE}} guide on the site. If it still has untranslated strings, it is the most useful file to pick. + +If you are working in one of our development sprints at a conference, someone from the pyOpenSci team will be available to help you get set up. + +### Claim your lines + +To keep our work from overlapping, claim a range of lines before you start: + +1. Read the comments below to see which lines are already taken. +2. Add a comment with the lines you will translate. You can copy this: + > I'm working on lines 1–100. +3. When your part is ready, open a Pull Request and link back to this issue. + +You can see line numbers when you open the file on GitHub. If you are not sure how much you can take on, start with a small range. You can always claim more later. + +When you link this issue from your Pull Request, please write `Part of #{{ISSUE_NUMBER}}` rather than `Closes #{{ISSUE_NUMBER}}`. Several people are translating this file, so it should stay open until every line is done. + +### Resources + +- [Translation Guide](https://www.pyopensci.org/python-package-guide/TRANSLATING.html) — the full workflow and how to set up your local environment +- [Editing the Translation Files](https://www.pyopensci.org/python-package-guide/TRANSLATING.html#editing-the-translation-files) — what a `.po` entry looks like, and tools that help you edit one +- [Frequently Asked Questions (FAQ)](https://www.pyopensci.org/python-package-guide/TRANSLATING.html#frequently-asked-questions-faq) — common questions answered + +### This file, as of {{STATS_DATE}} + +**If you come across a string marked `fuzzy`**, it already has a {{LANGUAGE}} translation, but that translation needs a second look to confirm it is correct. Usually this is because the English text changed after the string was translated, though a string can be marked fuzzy for other reasons too. Compare the translation against the English text above it. If it still says the right thing, simply remove the line with the **fuzzy** tag. If it does not, rewrite it and then remove the tag. The Translation Guide explains this further in [What happens when a string has changed in the original English text](https://www.pyopensci.org/python-package-guide/TRANSLATING.html#what-happens-when-a-string-has-changed-in-the-original-english-text). + +{{STATS_TABLE}} diff --git a/scripts/translation/parent-issue-template.md b/scripts/translation/parent-issue-template.md new file mode 100644 index 00000000..94d4042a --- /dev/null +++ b/scripts/translation/parent-issue-template.md @@ -0,0 +1,35 @@ +> _This issue is created automatically and should not be edited directly._ + +We are translating the Python Package Guide into {{LANGUAGE}}, and we need new contributors. If you speak {{LANGUAGE}} and you are new to open source, this is a great place to start! + +### What you will be doing + +The guide is divided into sections. For each section, the English text is stored in a `.po` file inside `./locales/{{LOCALE}}/LC_MESSAGES`. Next to each English string, there is a space to write the {{LANGUAGE}} translation. + +### Getting started + +Read the [Translation Guide](https://www.pyopensci.org/python-package-guide/TRANSLATING.html) first. It explains the workflow and how to set up your local environment. + +New to open source? You can also work entirely from the GitHub website. Fork the repository into your account, make your changes on your copy, and open a Pull Request. Two parts of the Translation Guide are worth reading first: [Editing the Translation Files](https://www.pyopensci.org/python-package-guide/TRANSLATING.html#editing-the-translation-files), which shows what a `.po` entry looks like, and the [Frequently Asked Questions (FAQ)](https://www.pyopensci.org/python-package-guide/TRANSLATING.html#frequently-asked-questions-faq). + +If you are working in one of our development sprints at a conference, someone from the pyOpenSci team will be available to help you get set up. + +### Pick a file and claim your work + +Each file in the table below has its own issue. Click a file name to open it. There, leave a comment claiming a range of lines to work on, so your work does not overlap with anyone else's. Read the existing comments first to see which lines are already taken. + +Look at the untranslated column — a file with a smaller number there is an easier place to start. + +**Not sure where to start?** If `index.po` still has untranslated strings, start there. Finishing it is what lets us publish this language, so it is the most useful file to work on when you don't have a particular section in mind. + +### See an example + +{{EXAMPLE_SECTION}} + +### Translation status as of {{STATS_DATE}} + +The table shows the number of strings in each file. + +**If you come across a string marked `fuzzy`**, it already has a {{LANGUAGE}} translation, but that translation needs a second look to confirm it is correct. Usually this is because the English text changed after the string was translated, though a string can be marked fuzzy for other reasons too. Compare the translation against the English text above it. If it still says the right thing, simply remove the line with the **fuzzy** tag. If it does not, rewrite it and then remove the tag. The Translation Guide explains this further in [What happens when a string has changed in the original English text](https://www.pyopensci.org/python-package-guide/TRANSLATING.html#what-happens-when-a-string-has-changed-in-the-original-english-text). + +{{STATS_TABLE}} diff --git a/scripts/translation/refresh_translation_issues.py b/scripts/translation/refresh_translation_issues.py new file mode 100644 index 00000000..4dd54471 --- /dev/null +++ b/scripts/translation/refresh_translation_issues.py @@ -0,0 +1,534 @@ +#!/usr/bin/env python +"""Update the GitHub issues that coordinate translation with fresh stats. + +Run by the GitHub Action ``refresh-translation-issues.yml`` whenever a ``.po`` +file changes on ``main``. Note: all locale issues are checked whenever any ``.po`` file +changes, not just the ones for the locale that changed in the last commit. This simplifies +the workflow without major cost since issues that do not need updating are skipped, and +ensures that the stats are always up to date and the bodies always remain consistent. + +Each language has one parent issue that serves as an index for the translation work. It +contains a table with the stats for each ``.po`` file, which are represented by +sub-issues containing each file's stats where contributors can claim line ranges. + +Every time a ``.po`` file changes the stats in these issues need to be updated, so +this script recreates the body each issue should have, compares it against +the current body it has, and returns for update only the ones that need to change. + +Usage:: + + python refresh_translation_issues.py ISSUES_JSON BODIES_JSON REPORT_MD + +``ISSUES_JSON`` maps each locale to the open issues carrying its ``lang-XX`` +label, as ``{"es": [{"number": 686, "title": ..., "body": ...}, ...]}``. +``BODIES_JSON`` receives the issues to rewrite, as ``[{"number", "body"}]``. +``REPORT_MD`` receives the comment to post, and is left empty when there is +nothing a maintainer has to do. + +If the scripts encounters discrepancies with what it expects, it reports back to +the action, which then notifies maintainers by making a comment on the translation +maintenance issue. + +The following situations are reported: + +- An issue's hidden HTML marker contradicts its title. The issue is skipped: one of the + two is wrong, and guessing would overwrite whatever it really holds. +- Two issues claim the same role. The lowest-numbered one is updated and the + other needs closing or renaming. +- A sub-issue tracks a catalog the guide no longer translates. Left alone. +- A ``.po`` file has no sub-issue. The parent lists it without a link until + someone opens one. +- Sub-issues exist but no parent matched. The whole locale is skipped, since a + sub-issue's body links back to its parent. +- An existing locale has ``.po`` files but no open issues (e.g. ``ja``) +- A parent issue exists for the locale, but sub-issues do not (e.g. ``de``) +- A locale has no ``.po`` files the guide still translates, so there is nothing + to render a body from. + +""" + +from __future__ import annotations + +import json +import re +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import NamedTuple + +from babel import Locale +from stats import PoFileStats, TranslationStats, get_translation_stats + +HERE = Path(__file__).resolve().parent +PARENT_TEMPLATE_PATH = HERE / "parent-issue-template.md" +CHILD_TEMPLATE_PATH = HERE / "child-issue-template.md" + +REPO = "pyOpenSci/python-package-guide" + +# The string that identifies the parent issues in the maps below. Sub-issues are +# identified by the name of the `.po` files they track. +PARENT_ROLE = "parent" + +# Below this percentage nothing in a language is finished enough to point a +# beginner to, so the example falls back to a default language that has one. +EXAMPLE_MIN_PERCENTAGE = 50.0 +EXAMPLE_FALLBACK = ("es", "index.po") + +PARENT_TABLE_HEADER = ( + "| File | Status | Translated | Fuzzy | Untranslated |\n" + "| :--- | -----: | ---------: | ----: | -----------: |" +) +CHILD_TABLE_HEADER = ( + "| Status | Translated | Fuzzy | Untranslated |\n" + "| -----: | ---------: | ----: | -----------: |" +) + +# The only date in the body is the one stamped on its stats heading. +DATE_STAMP = re.compile(r"\d{4}-\d{2}-\d{2}") + +PARENT_TITLE = re.compile(r"^Help translate the Python Packaging Guide into ") +CHILD_TITLE = re.compile(r"^Translate `([^`]+\.po)` into ") + +# A hidden HTML comment marker we add to the issue bodies to verify later we +# are reading the right issue. +MARKER = re.compile(r"") + + +class Message(NamedTuple): + """A message we want to post as a comment to the Translation Maintenance issue. + + Use `actionable=True` for messages that maintainers have to take action on. This + signals to the GitHub action that these messages should be posted to the translation + maintenance issue and end up in the #translations channel in Slack. + """ + + text: str + actionable: bool + + +# --------------------------------------------------------------------------- # +# Rendering Markdown +# --------------------------------------------------------------------------- # + + +def by_filename(locale_stats: dict[str, PoFileStats]) -> dict[str, PoFileStats]: + """Re-key stats by filename. + + :mod:`stats` keys catalogs by stem (``index``) while the issue bodies and + their titles all use the filename (``index.po``). + """ + return {f"{stem}.po": counts for stem, counts in locale_stats.items()} + + +def display_order(files: dict[str, PoFileStats]) -> list[str]: + """Sort the way the tables read: most complete first. + + Ties break on the untranslated count, so the file that is quickest to finish + comes first. The parent issue tells contributors to choose by that column. + """ + + def rank(name: str) -> tuple[float, int]: + return -files[name]["percentage"], files[name]["untranslated"] + + return sorted(files, key=rank) + + +def choose_example(code: str, files: dict[str, PoFileStats]) -> tuple[str, str]: + """The file to point a beginner to, as a ``(locale, filename)`` pair.""" + best = display_order(files)[0] + if files[best]["percentage"] < EXAMPLE_MIN_PERCENTAGE: + return EXAMPLE_FALLBACK + return code, best + + +def language_name(code: str) -> str: + """Use babel to get the language name, e.g.: ``"es"`` to ``"Spanish"``.""" + return Locale.parse(code).get_display_name("en") + + +def po_url(locale: str, filename: str) -> str: + return ( + f"https://github.com/{REPO}/blob/main/locales/{locale}/LC_MESSAGES/{filename}" + ) + + +def issue_url(number: int) -> str: + return f"https://github.com/{REPO}/issues/{number}" + + +def _row(*cells: str) -> str: + return "| " + " | ".join(cells) + " |" + + +def _count_cells(counts: PoFileStats | dict[str, float]) -> list[str]: + return [ + f"{counts['percentage']:.1f}%", + str(counts["translated"]), + str(counts["fuzzy"]), + str(counts["untranslated"]), + ] + + +def _total_row(files: dict[str, PoFileStats]) -> str: + total = sum(counts["total"] for counts in files.values()) + summed = { + key: sum(counts[key] for counts in files.values()) + for key in ("translated", "fuzzy", "untranslated") + } + summed["percentage"] = summed["translated"] / total * 100 if total else 0.0 + return _row("**Total**", *(f"**{cell}**" for cell in _count_cells(summed))) + + +def render_parent_table( + files: dict[str, PoFileStats], subissues: dict[str, int] +) -> str: + """The parent issue's table: every po file, linked to its own sub-issue.""" + rows = [PARENT_TABLE_HEADER] + for name in display_order(files): + number = subissues.get(name) + # A po file without an open sub-issue is named but not linked. It will be + # reported by the GitHub action in the maintenance issue. + cell = f"[`{name}`]({issue_url(number)})" if number else f"`{name}`" + rows.append(_row(cell, *_count_cells(files[name]))) + rows.append(_total_row(files)) + return "\n".join(rows) + + +def render_child_table(counts: PoFileStats) -> str: + """A sub-issue's table with the stats for the file it covers.""" + return "\n".join([CHILD_TABLE_HEADER, _row(*_count_cells(counts))]) + + +def example_section(example: tuple[str, str], code: str) -> str: + """The parent issue's "See an example" paragraph. + + We don't put the percentage to avoid it being out of date if the example file + changes. + """ + locale, name = example + link = f"[`{name}`]({po_url(locale, name)})" + if locale == code: + return ( + "Want to see what a translated file looks like? Look at " + f"{link}, the file that is furthest along in this language." + ) + # If we use another language as example, we say which. + return ( + "Want to see what a translated file looks like? Here is one from the " + f"{language_name(locale)} translation: {link}." + ) + + +def marker(locale: str, role: str) -> str: + """A hidden marker we add to the issue body to save its locale and role.""" + return f"" + + +def fill_template(template: str, values: dict[str, str]) -> str: + """Substitute the ``{{PLACEHOLDER}}`` values.""" + for name, value in values.items(): + template = template.replace(f"{{{{{name}}}}}", value) + return template + + +# --------------------------------------------------------------------------- # +# Working out which issue is which +# --------------------------------------------------------------------------- # + + +def role_from_title(title: str) -> str | None: + """Identify the issue role (parent or the po file for a sub-issue) using the title.""" + if PARENT_TITLE.match(title): + return PARENT_ROLE + found = CHILD_TITLE.match(title) + return found.group(1) if found else None + + +def role_from_body_marker(body: str) -> str | None: + """Find the ``/`` from the body marker if there is one.""" + found = MARKER.search(body) + return found.group(1) if found else None + + +def _listed(numbers: list[int]) -> str: + return ", ".join(f"#{number}" for number in numbers) + + +def match_issues( + locale: str, issues: list[dict] +) -> tuple[dict[str, dict], list[Message]]: + """Sort a locale's issues into ``{role: issue}``, saying what was left out. + + Issues are matched on their title, then confirmed against the hidden marker in + their body to prevent rewriting at the wrong issue. + """ + matched: dict[str, dict] = {} + messages: list[Message] = [] + ignored: list[int] = [] + # If two issues claim the same file, the older one wins (lowest number) + for issue in sorted(issues, key=lambda issue: issue["number"]): + number = issue["number"] + role = role_from_title(issue["title"]) + if role is None: + ignored.append(number) + continue + claimed = role_from_body_marker(issue.get("body") or "") + if claimed is not None and claimed != f"{locale}/{role}": + messages.append( + Message( + f"#{number} is titled as the issue for `{locale}/{role}`, but " + f"its body is marked `{claimed}`. It was left alone: one of " + "the two is wrong, and guessing which would overwrite " + "whatever the issue really holds.", + actionable=True, + ) + ) + continue + if role in matched: + messages.append( + Message( + f"#{matched[role]['number']} and #{number} both look like the " + f"issue for `{locale}/{role}`. The first was updated; close " + "or rename the other.", + actionable=True, + ) + ) + continue + matched[role] = issue + if ignored: + messages.append( + Message( + f"`{locale}`: {len(ignored)} labelled issue(s) are not translation " + f"trackers and were ignored: {_listed(ignored)}.", + actionable=False, + ) + ) + return matched, messages + + +# --------------------------------------------------------------------------- # +# Comparing against what is already on GitHub +# --------------------------------------------------------------------------- # + + +def normalize_body(body: str) -> str: + """Normalize the body for comparison. + + GitHub hands bodies back with CRLF line endings and without any blank line + the body ended on. The stats date will always changes, so it should not count + as a difference. + """ + return DATE_STAMP.sub("", body.replace("\r\n", "\n").strip()) + + +def is_unchanged(current: str, planned: str) -> bool: + """Whether an issue already says what we would write.""" + return normalize_body(current) == normalize_body(planned) + + +# --------------------------------------------------------------------------- # +# Putting it together +# --------------------------------------------------------------------------- # + + +def render_bodies( + locale: str, + files: dict[str, PoFileStats], + matched: dict[str, dict], + today: str, +) -> dict[int, str]: + """Every issue this run would write for one locale, keyed by issue number.""" + language = language_name(locale) + example = choose_example(locale, files) + parent = matched[PARENT_ROLE] + subissues = { + role: issue["number"] for role, issue in matched.items() if role != PARENT_ROLE + } + + def finish(body: str, role: str) -> str: + return f"{body.rstrip()}\n\n{marker(locale, role)}" + + bodies = { + parent["number"]: finish( + fill_template( + PARENT_TEMPLATE_PATH.read_text(encoding="utf-8"), + { + "LANGUAGE": language, + "LOCALE": locale, + "EXAMPLE_SECTION": example_section(example, locale), + "STATS_DATE": today, + "STATS_TABLE": render_parent_table(files, subissues), + }, + ), + PARENT_ROLE, + ) + } + + child_template = CHILD_TEMPLATE_PATH.read_text(encoding="utf-8") + for name, number in subissues.items(): + if name not in files: + continue # its English page is gone. This will be reported in the maintenance issue. + bodies[number] = finish( + fill_template( + child_template, + { + "LANGUAGE": language, + "FILENAME": name, + "FILE_URL": po_url(locale, name), + "MAIN_ISSUE_URL": issue_url(parent["number"]), + "ISSUE_NUMBER": str(number), + "STATS_DATE": today, + "STATS_TABLE": render_child_table(files[name]), + }, + ), + name, + ) + return bodies + + +def _bookkeeping( + locale: str, files: dict[str, PoFileStats], subissue_roles: set[str] +) -> list[Message]: + """Alert on changes between the `.po` files on disk and the issues on GitHub.""" + messages = [] + for name in sorted(subissue_roles - set(files)): + messages.append( + Message( + f"`{locale}`: the issue for `{name}` tracks a catalog the guide no " + "longer translates. Its body was left alone. Close it if that page " + "is gone for good or migrate the strings into whichever catalog " + "covers it now.", + actionable=True, + ) + ) + for name in sorted(set(files) - subissue_roles): + messages.append( + Message( + f"`{locale}`: `{name}` has no sub-issue, so the parent issue lists " + "it without a link. Open one titled " + # Double backticks, because the title itself contains backticks + # and a backslash inside a code span renders as a backslash. + f"``Translate `{name}` into {language_name(locale)}`` with the " + f"`lang-{locale.upper()}` label, then re-run this workflow.", + actionable=True, + ) + ) + return messages + + +def updates( + found: dict[str, list[dict]], + stats: TranslationStats | None = None, + today: str | None = None, +) -> tuple[list[dict], list[Message]]: + """Updates the issues that need new stats.""" + if stats is None: + stats = get_translation_stats() + if today is None: + today = datetime.now(timezone.utc).strftime("%Y-%m-%d") + + changed: list[dict] = [] + messages: list[Message] = [] + for locale in sorted(found): + matched, notes = match_issues(locale, found[locale]) + messages.extend(notes) + + files = by_filename(stats.get(locale, {})) + if not files: + messages.append( + Message( + f"`{locale}` has no .po file catalogs the guide still translates.", + actionable=True, + ) + ) + continue + if not matched: + messages.append( + Message( + f"`{locale}` has no translation issues. This workflow only " + "updates issues, please create them manually.", + actionable=True, + ) + ) + continue + + subissue_roles = set(matched) - {PARENT_ROLE} + if PARENT_ROLE not in matched: + messages.append( + Message( + f"`{locale}` has {len(subissue_roles)} sub-issue(s) but no " + "parent issue, and a sub-issue's body links back to its " + "parent. The whole language was skipped. Open the parent, " + "titled `Help translate the Python Packaging Guide into " + f"{language_name(locale)}` with the `lang-{locale.upper()}` " + "label.", + actionable=True, + ) + ) + continue + if not subissue_roles: + messages.append( + Message( + f"`{locale}` has a parent issue but no sub-issues, so only the " + "parent was refreshed. Open one sub-issue per `.po` file so " + "contributors have somewhere to claim line ranges.", + actionable=True, + ) + ) + else: + messages.extend(_bookkeeping(locale, files, subissue_roles)) + + current = { + issue["number"]: issue.get("body") or "" for issue in matched.values() + } + for number, body in render_bodies(locale, files, matched, today).items(): + if not is_unchanged(current[number], body): + changed.append({"number": number, "body": body}) + return changed, messages + + +def render_report(messages: list[Message]) -> str: + """The comment for the translation maintenance issue.""" + needed = [message.text for message in messages if message.actionable] + if not needed: + return "" + listed = "\n".join(f"- {text}" for text in needed) + return f"""### Some translation issues need a look + +The `.po` files changed, so the issues tracking them were rewritten with the +current numbers. These could not be handled automatically: + +{listed} + +Nothing was closed, retitled or relabelled: this workflow only ever rewrites the +body of an issue. +""" + + +def main(argv: list[str]) -> int: + """Render the bodies for the issues described in ``argv[1]``. + + Always succeeds to prevent a red mark on `main` from reporting a bookkeeping + problem as if the translation that triggered this run were at fault. + """ + if len(argv) != 4: + print( + f"usage: {Path(argv[0]).name} ISSUES_JSON BODIES_JSON REPORT_MD", + file=sys.stderr, + ) + return 0 + _, issues_path, bodies_path, report_path = argv + found = json.loads(Path(issues_path).read_text(encoding="utf-8")) + changed, messages = updates(found) + + Path(bodies_path).write_text(json.dumps(changed), encoding="utf-8") + Path(report_path).write_text(render_report(messages), encoding="utf-8") + + for message in messages: + print(message.text, file=sys.stderr) + print(f"\n{len(changed)} issue(s) need rewriting.", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/scripts/translation/test_refresh_translation_issues.py b/scripts/translation/test_refresh_translation_issues.py new file mode 100644 index 00000000..dc4f6067 --- /dev/null +++ b/scripts/translation/test_refresh_translation_issues.py @@ -0,0 +1,411 @@ +"""Tests for the workflow that keeps the translation issues up to date. + +The tables and the example paragraph are checked against the issues currently +live on GitHub, so a change to the wording or the layout fails a test here +rather than quietly rewriting forty-odd issues. +""" + +from __future__ import annotations + +import re + +import refresh_translation_issues as refresh + + +def counts(percentage, translated, fuzzy, untranslated): + """One file's stats, in the order the tables read them.""" + return { + "total": translated + fuzzy + untranslated, + "translated": translated, + "fuzzy": fuzzy, + "untranslated": untranslated, + "percentage": percentage, + "stale": False, + "missing": 0, + } + + +def issue(number, title, body=""): + return {"number": number, "title": title, "body": body} + + +# Spanish as issue #686 reported it, keyed by filename the way the bodies are. +SPANISH = { + "index.po": counts(81.0, 81, 11, 8), + "package-structure-code.po": counts(66.1, 601, 161, 147), + "documentation.po": counts(31.0, 179, 14, 385), + "tests.po": counts(23.3, 64, 75, 136), + "tutorials.po": counts(15.0, 179, 0, 1014), + "TRANSLATING.po": counts(0.0, 0, 0, 109), + "CONTRIBUTING.po": counts(0.0, 0, 0, 126), + "maintain-automate.po": counts(0.0, 0, 0, 292), +} + +SPANISH_SUBISSUES = { + "index.po": 687, + "package-structure-code.po": 688, + "tests.po": 689, + "documentation.po": 690, + "tutorials.po": 691, + "TRANSLATING.po": 692, + "maintain-automate.po": 693, + "CONTRIBUTING.po": 694, +} + +# A language where nothing has been started, so no file can be the example. +UNSTARTED = { + "index.po": counts(0.0, 0, 0, 100), + "tests.po": counts(0.0, 0, 0, 275), +} + +PARENT_TITLE = "Help translate the Python Packaging Guide into Spanish" + +# Two files is enough for the end-to-end tests, and keeps the fixtures readable. +# `stats` keys catalogs by stem, so this is the shape `updates` is handed. +TWO_FILES = { + "es": {"index": counts(81.0, 81, 11, 8), "tests": counts(23.3, 64, 75, 136)} +} + + +# --------------------------------------------------------------------------- # +# Tables +# --------------------------------------------------------------------------- # + + +def test_the_parent_table_matches_the_one_on_issue_686(): + assert ( + refresh.render_parent_table(SPANISH, SPANISH_SUBISSUES) + == """\ +| File | Status | Translated | Fuzzy | Untranslated | +| :--- | -----: | ---------: | ----: | -----------: | +| [`index.po`](https://github.com/pyOpenSci/python-package-guide/issues/687) | 81.0% | 81 | 11 | 8 | +| [`package-structure-code.po`](https://github.com/pyOpenSci/python-package-guide/issues/688) | 66.1% | 601 | 161 | 147 | +| [`documentation.po`](https://github.com/pyOpenSci/python-package-guide/issues/690) | 31.0% | 179 | 14 | 385 | +| [`tests.po`](https://github.com/pyOpenSci/python-package-guide/issues/689) | 23.3% | 64 | 75 | 136 | +| [`tutorials.po`](https://github.com/pyOpenSci/python-package-guide/issues/691) | 15.0% | 179 | 0 | 1014 | +| [`TRANSLATING.po`](https://github.com/pyOpenSci/python-package-guide/issues/692) | 0.0% | 0 | 0 | 109 | +| [`CONTRIBUTING.po`](https://github.com/pyOpenSci/python-package-guide/issues/694) | 0.0% | 0 | 0 | 126 | +| [`maintain-automate.po`](https://github.com/pyOpenSci/python-package-guide/issues/693) | 0.0% | 0 | 0 | 292 | +| **Total** | **30.8%** | **1104** | **261** | **2217** |""" + ) + + +def test_the_child_table_matches_the_one_on_issue_687(): + assert ( + refresh.render_child_table(SPANISH["index.po"]) + == """\ +| Status | Translated | Fuzzy | Untranslated | +| -----: | ---------: | ----: | -----------: | +| 81.0% | 81 | 11 | 8 |""" + ) + + +def test_a_file_without_a_sub_issue_is_listed_but_not_linked(): + """A new English page gets a catalog before anyone opens its issue.""" + files = {"index.po": counts(81.0, 81, 11, 8), "brand-new.po": counts(0.0, 0, 0, 12)} + table = refresh.render_parent_table(files, {"index.po": 687}) + assert "| `brand-new.po` | 0.0% | 0 | 0 | 12 |" in table + assert "brand-new.po](" not in table + + +def test_files_are_ordered_most_complete_first(): + assert refresh.display_order(SPANISH)[:3] == [ + "index.po", + "package-structure-code.po", + "documentation.po", + ] + + +def test_a_tie_is_broken_by_the_untranslated_count(): + """Of two files at the same percentage, the shorter one comes first.""" + files = {"long.po": counts(0.0, 0, 0, 292), "short.po": counts(0.0, 0, 0, 109)} + assert refresh.display_order(files) == ["short.po", "long.po"] + + +def test_the_total_row_rounds_rather_than_truncating(): + """126 of 3620 is 3.48%, which has to read as 3.5% and not 3.4%.""" + files = {"a.po": counts(0.0, 126, 0, 3494)} + assert "**3.5%**" in refresh._total_row(files) + + +def test_stats_are_re_keyed_from_stems_to_filenames(): + assert refresh.by_filename({"index": SPANISH["index.po"]}) == { + "index.po": SPANISH["index.po"] + } + + +# --------------------------------------------------------------------------- # +# The example file +# --------------------------------------------------------------------------- # + + +def test_the_example_is_the_most_complete_file(): + assert refresh.choose_example("es", SPANISH) == ("es", "index.po") + + +def test_the_example_falls_back_to_spanish_when_nothing_is_ready(): + assert refresh.choose_example("de", UNSTARTED) == ("es", "index.po") + + +def test_the_example_never_quotes_a_percentage(): + """The number would go stale on any run that did not rewrite this issue.""" + section = refresh.example_section(("es", "index.po"), "es") + assert "%" not in section + assert "locales/es/LC_MESSAGES/index.po" in section + + +def test_a_borrowed_example_names_the_language_it_came_from(): + """Otherwise it reads as a claim about the language being tracked.""" + section = refresh.example_section(("es", "index.po"), "de") + assert "Spanish" in section + + +# --------------------------------------------------------------------------- # +# Working out which issue is which +# --------------------------------------------------------------------------- # + + +def test_a_title_says_which_issue_it_is(): + assert refresh.role_from_title(PARENT_TITLE) == "parent" + assert refresh.role_from_title("Translate `index.po` into Spanish") == "index.po" + + +def test_an_unrelated_labelled_issue_is_ignored_quietly(): + """#522 is a bug report that carries `lang-JA` like any Japanese issue.""" + title = "The emphasized text is not rendering correctly in Japanese document" + assert refresh.role_from_title(title) is None + + matched, messages = refresh.match_issues("ja", [issue(522, title)]) + assert matched == {} + assert not any(message.actionable for message in messages) + + +def test_an_issue_with_no_marker_is_accepted(): + """Every issue that existed before this script did carries no marker.""" + matched, messages = refresh.match_issues("es", [issue(686, PARENT_TITLE)]) + assert set(matched) == {"parent"} + assert messages == [] + + +def test_a_marker_that_contradicts_the_title_is_reported_and_skipped(): + found = [ + issue( + 687, "Translate `index.po` into Spanish", refresh.marker("es", "tests.po") + ) + ] + matched, messages = refresh.match_issues("es", found) + assert matched == {} + assert [message.actionable for message in messages] == [True] + assert "#687" in messages[0].text and "es/tests.po" in messages[0].text + + +def test_a_marker_from_another_locale_is_caught_too(): + found = [ + issue( + 687, "Translate `index.po` into Spanish", refresh.marker("pt", "index.po") + ) + ] + matched, messages = refresh.match_issues("es", found) + assert matched == {} + assert messages[0].actionable + + +def test_the_lowest_numbered_duplicate_wins(): + """The older issue is the one contributors have been commenting on.""" + title = "Translate `index.po` into Spanish" + matched, messages = refresh.match_issues( + "es", [issue(900, title), issue(687, title)] + ) + assert matched["index.po"]["number"] == 687 + assert messages[0].actionable + assert "#687" in messages[0].text and "#900" in messages[0].text + + +# --------------------------------------------------------------------------- # +# Rendering the body +# --------------------------------------------------------------------------- # + + +def rendered(): + """The bodies for a Spanish parent and one sub-issue.""" + matched = { + "parent": issue(686, PARENT_TITLE), + "index.po": issue(687, "Translate `index.po` into Spanish"), + } + return refresh.render_bodies("es", SPANISH, matched, "2026-08-04") + + +def test_every_rendered_body_ends_with_its_own_marker(): + bodies = rendered() + assert bodies[686].endswith("") + assert bodies[687].endswith("") + + +def test_no_placeholder_survives_into_a_rendered_body(): + for body in rendered().values(): + assert "{{" not in body + + +def test_a_sub_issue_links_back_to_its_parent_and_to_itself(): + body = rendered()[687] + assert "https://github.com/pyOpenSci/python-package-guide/issues/686" in body + assert "Part of #687" in body + + +def test_the_templates_carry_the_placeholders_the_script_fills(): + """A template edit that renames a placeholder has to fail a test. + + Nothing else reads these files, so a typo would otherwise reach GitHub as a + literal `{{LANGAUGE [sic]}}` in forty issues. + Note: the [sic] is required to avoid triggering a codespell failure in CI. + """ + parent = refresh.PARENT_TEMPLATE_PATH.read_text(encoding="utf-8") + child = refresh.CHILD_TEMPLATE_PATH.read_text(encoding="utf-8") + assert set(re.findall(r"{{(\w+)}}", parent)) == { + "LANGUAGE", + "LOCALE", + "EXAMPLE_SECTION", + "STATS_DATE", + "STATS_TABLE", + } + assert set(re.findall(r"{{(\w+)}}", child)) == { + "LANGUAGE", + "FILENAME", + "FILE_URL", + "MAIN_ISSUE_URL", + "ISSUE_NUMBER", + "STATS_DATE", + "STATS_TABLE", + } + # The marker is appended by the script, so a comment in a template would be + # the only other thing that could cause a false positive. + assert "