Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .github/actionlint.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
paths:
.github/workflows/update-and-build.yml:
ignore:
# Remove once https://github.com/rhysd/actionlint/pull/732 is released.
- ^reusable workflow call "\$/\.github/workflows/spellcheck\.yml" at "uses" is not following
the format
159 changes: 159 additions & 0 deletions .github/workflows/spellcheck.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
name: spellcheck

on:
pull_request:
workflow_dispatch:
workflow_call:
inputs:
advisory:
description: Report failures without failing the calling workflow
type: boolean
default: true
version:
description: CPython documentation version
required: true
type: string
translations-ref:
description: Translation branch updated by the calling workflow
required: true
type: string
secrets:
TX_TOKEN:
description: Transifex API token with access to the Polish terminology glossary
required: true

permissions:
contents: read

jobs:
pospell:
runs-on: ubuntu-latest
continue-on-error: ${{ inputs.advisory || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }}
timeout-minutes: 15
defaults:
run:
working-directory: cpython/Doc
steps:
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: 3
- uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
- name: Check out workflow tools
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
path: spellcheck-tools
sparse-checkout: |
/spellcheck_report.py
/manage_translation.py
sparse-checkout-cone-mode: false
persist-credentials: false
- name: Check out CPython documentation
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: python/cpython
ref: ${{ inputs.version || github.base_ref || github.ref_name }}
path: cpython
persist-credentials: false
- name: Check out translations
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ inputs.translations-ref }}
path: cpython/Doc/locales/pl/LC_MESSAGES
persist-credentials: false
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y curl hunspell hunspell-pl unzip
make venv
# Pospell needs newer Docutils than the Sphinx used by older branches.
uv venv "$RUNNER_TEMP/spellcheck-venv"
# Pospell PR 24 provides --report; switch to a release when available.
uv pip install --python "$RUNNER_TEMP/spellcheck-venv/bin/python" \
sphinx-intl transifex-python six \
'pospell @ git+https://git.afpy.org/maciek/pospell.git@feat/json-reports'
- name: Prepare UTF-8 Polish dictionary
# The packaged dictionary uses ISO-8859-2, which cannot represent all prose.
run: |
mkdir -p "$RUNNER_TEMP/hunspell-utf8"
dictionary_encoding=$(sed -n 's/^SET //p' /usr/share/hunspell/pl_PL.aff)
iconv -f "$dictionary_encoding" -t UTF-8 /usr/share/hunspell/pl_PL.aff \
> "$RUNNER_TEMP/hunspell-utf8/pl_PL.aff"
sed -i 's/^SET .*/SET UTF-8/' "$RUNNER_TEMP/hunspell-utf8/pl_PL.aff"
iconv -f "$dictionary_encoding" -t UTF-8 /usr/share/hunspell/pl_PL.dic \
> "$RUNNER_TEMP/hunspell-utf8/pl_PL.dic"
- name: Download SJP Polish dictionary
run: |
curl --fail --location --retry 3 \
https://sjp.pl/sl/ort/sjp-hunspell-pl-20261101.zip \
--output "$RUNNER_TEMP/sjp-hunspell.zip"
echo "a29b40d80a2501ba1ad84fa17f7d177e2ebe313082f74f837a77b3135c05340c $RUNNER_TEMP/sjp-hunspell.zip" \
| sha256sum --check --strict
# SJP wraps the dictionary files in a second ZIP archive.
unzip -p "$RUNNER_TEMP/sjp-hunspell.zip" pl_PL.zip \
> "$RUNNER_TEMP/sjp-pl_PL.zip"
unzip -p "$RUNNER_TEMP/sjp-pl_PL.zip" pl_PL.aff \
> "$RUNNER_TEMP/sjp-pl_PL.aff"
unzip -p "$RUNNER_TEMP/sjp-pl_PL.zip" pl_PL.dic \
> "$RUNNER_TEMP/sjp-pl_PL.dic"
dictionary_encoding=$(sed -n 's/^SET //p' "$RUNNER_TEMP/sjp-pl_PL.aff")
iconv -f "$dictionary_encoding" -t UTF-8 "$RUNNER_TEMP/sjp-pl_PL.aff" \
> "$RUNNER_TEMP/hunspell-utf8/pl_SJP.aff"
sed -i 's/^SET .*/SET UTF-8/' "$RUNNER_TEMP/hunspell-utf8/pl_SJP.aff"
iconv -f "$dictionary_encoding" -t UTF-8 "$RUNNER_TEMP/sjp-pl_PL.dic" \
> "$RUNNER_TEMP/hunspell-utf8/pl_SJP.dic"
- name: Download Polish terminology glossary
working-directory: spellcheck-tools
env:
TX_TOKEN: ${{ secrets.TX_TOKEN }}
run: >
"$RUNNER_TEMP/spellcheck-venv/bin/python"
manage_translation.py fetch_glossary
--output "$RUNNER_TEMP/hunspell-glossary.dic"
- name: Generate POT files without literal blocks
run: |
if make -n gettext >/dev/null 2>&1; then
make gettext SPHINXOPTS="-D gettext_additional_targets=index -D gettext_compact=0"
else
# CPython 3.10 and 3.11 do not have a dedicated gettext target.
make build BUILDER=gettext SPHINXOPTS="-D gettext_additional_targets=index -D gettext_compact=0"
fi
- name: Record original translation checksums
run: >
python "$GITHUB_WORKSPACE/spellcheck-tools/spellcheck_report.py" snapshot
--translations locales/pl/LC_MESSAGES --cpython .
--output "$RUNNER_TEMP/spellcheck-source.json"
- name: Update temporary PO files
# Drop obsolete entries so removed literal blocks are not spellchecked.
run: >
"$RUNNER_TEMP/spellcheck-venv/bin/sphinx-intl" update -p build/gettext -d locales
-l pl
--no-obsolete
- name: Check Polish spelling
id: spelling
env:
DICPATH: ${{ runner.temp }}/hunspell-utf8
run: >
"$RUNNER_TEMP/spellcheck-venv/bin/pospell" --language pl_PL,pl_SJP
--personal-dict "$RUNNER_TEMP/hunspell-glossary.dic"
--glob 'locales/pl/LC_MESSAGES/**/*.po'
--report "$RUNNER_TEMP/pospell.json"
--report-root locales/pl/LC_MESSAGES
- name: Prepare spelling report
id: report
if: ${{ !cancelled() && (steps.spelling.outcome == 'success' || steps.spelling.outcome == 'failure') }}
env:
DOCS_VERSION: ${{ inputs.version || github.base_ref || github.ref_name }}
run: >
python "$GITHUB_WORKSPACE/spellcheck-tools/spellcheck_report.py" prepare
--input "$RUNNER_TEMP/pospell.json"
--snapshot "$RUNNER_TEMP/spellcheck-source.json"
--docs-version "$DOCS_VERSION"
--output "$RUNNER_TEMP/spellcheck.json"
- name: Upload spelling report
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: ${{ !cancelled() && steps.report.outcome == 'success' }}
with:
name: spellcheck-pl
path: ${{ runner.temp }}/spellcheck.json
if-no-files-found: error
# PO updates exist only in this disposable checkout; nothing is committed.
14 changes: 14 additions & 0 deletions .github/workflows/update-and-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,20 @@ jobs:
branch: ${{ matrix.version }}
github_token: ${{ secrets.GITHUB_TOKEN }}

spellcheck:
needs: [update]
# These are the runs in which update publishes the fetched translations.
if: github.event_name == 'schedule' || github.ref_name == github.event.repository.default_branch
permissions:
contents: read
uses: $/.github/workflows/spellcheck.yml
with:
advisory: true
version: ${{ github.ref_name }}
translations-ref: ${{ github.ref_name }}
secrets:
TX_TOKEN: ${{ secrets.TX_TOKEN }}

build:
runs-on: ubuntu-latest
strategy:
Expand Down
37 changes: 37 additions & 0 deletions SPELLCHECK_REPORT.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# Spellcheck reports

The spellcheck workflow publishes a `spellcheck-pl` Actions artifact containing
`spellcheck.json`. Pospell is installed from `maciek/pospell@feat/json-reports`
until a release includes JSON reports. See the
[pospell documentation](https://git.afpy.org/maciek/pospell/src/branch/feat/json-reports/README.md)
for the report schema, finding identities, translation fingerprints, and statuses.

Normal text diagnostics remain in the job log. The report is prepared and uploaded
even when spelling findings cause the check step to fail; the job is advisory by
default. If pospell cannot write its JSON or report preparation fails, no artifact
is published.

## Integration fields

`spellcheck_report.py` preserves pospell's schema-v1 findings and adds or adapts:

- `language`: `pl`, the translation language expected by consumers.
- `dictionaries`: pospell's dictionary names, currently `pl_PL` and `pl_SJP`.
- `repository`: the GitHub repository name, such as `python/python-docs-pl`.
- `docs_version`: the documentation branch checked, such as `3.15`.
- `run_url`: the Actions run URL.
- `translations_revision` and `cpython_revision`: the checked-out Git commits.

Catalog paths are relative to `locales/pl/LC_MESSAGES`, such as
`tutorial/guide.po`. Each catalog's `source_sha256` is replaced with the checksum
of the original PO, recorded before the temporary `sphinx-intl update`. This lets
consumers compare coverage with their repository checkout. Pospell's checksum of
the temporarily updated PO is retained as `scanned_sha256`.

Only catalogs present both in the original checkout and in pospell's report are
published; unscanned files are not added as clean. The scan covers prose and
excludes literal blocks through the temporary gettext update.

`potodo-docs-priority` downloads the artifact separately. It validates the source
run, selects complete reports for its requested branch, and checks local entry
identities and fingerprints. This workflow requires no dispatch to that project.
84 changes: 83 additions & 1 deletion manage_translation.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,15 @@
# * recreate_tx_config: recreate configuration for all resources.
# * warn_about_files_to_delete: lists files that are not available upstream
# * generate_commit_msg: generates commit message with co-authors
# * fetch_glossary: download Polish terminology as a Hunspell personal dictionary

import csv
import io
import os
import re
import sys
import time
import unicodedata
from argparse import ArgumentParser
from collections.abc import Iterable
from contextlib import chdir
Expand All @@ -25,12 +31,15 @@
from typing import Self
from warnings import warn

import requests
from polib import POFile, pofile
from transifex.api import transifex_api
from transifex.api.jsonapi.exceptions import JsonApiException

LANGUAGE = 'pl'
PROJECT_SLUG = 'python-newest'
VERSION = '3.14'
GLOSSARY_ID = 'o:python-doc:g:python_global'


def fetch():
Expand All @@ -46,6 +55,59 @@ def fetch():
_call(f'msgcat --no-location -o {file} {file}')


def _download_glossary(token: str, *, max_wait: float = 300) -> str:
transifex_api.setup(auth=token)
# The SDK supports new API resource types before adding dedicated classes.
download = transifex_api.new(
type='glossaries_async_downloads',
glossary=transifex_api.new(type='glossaries', id=GLOSSARY_ID),
language=transifex_api.Language(id=f'l:{LANGUAGE}'),
)
download.save()
deadline = time.monotonic() + max_wait
while not download.redirect:
status = download.attributes['status']
if status == 'failed':
raise ValueError('Transifex glossary export failed.')
if status not in {'pending', 'processing'}:
raise ValueError(f'Unexpected glossary export status: {status!r}')
if time.monotonic() >= deadline:
raise ValueError('Transifex glossary export timed out.')
time.sleep(3)
download.reload()
# Download the CSV without sending the API token to the signed storage URL.
export = requests.get(download.redirect, timeout=30)
export.raise_for_status()
return export.content.decode('utf-8-sig')


def _glossary_words(content: str) -> list[str]:
reader = csv.DictReader(io.StringIO(content))
translation_column = f'translation_{LANGUAGE}'
if not reader.fieldnames or translation_column not in reader.fieldnames:
raise ValueError('Transifex glossary CSV has no Polish translation column.')
words = {
word
for row in reader
for word in re.findall(
r'[^\W\d_]+', unicodedata.normalize('NFC', row[translation_column] or '')
)
}
if not words:
raise ValueError('Transifex glossary has no Polish terms.')
return sorted(words)


def fetch_glossary(output: Path):
"""Fetch Polish terminology and write a UTF-8 Hunspell personal dictionary."""
token = _get_tx_token().strip()
if not token:
raise ValueError('TX_TOKEN is required to download the Transifex glossary.')
words = _glossary_words(_download_glossary(token))
output.write_text(f'{len(words)}\n' + '\n'.join(words) + '\n', encoding='utf-8')
print(f'Added {len(words)} Polish glossary words to {output}.')


def _call(command: str):
if (return_code := call(command, shell=True)) != 0:
sys.exit(return_code)
Expand Down Expand Up @@ -224,10 +286,30 @@ def generate_commit_msg():
'recreate_tx_config',
'warn_about_files_to_delete',
'generate_commit_msg',
'fetch_glossary',
)

parser = ArgumentParser()
parser.add_argument('cmd', choices=RUNNABLE_SCRIPTS)
parser.add_argument(
'--output', type=Path, help='Hunspell dictionary path for fetch_glossary'
)
options = parser.parse_args()

eval(options.cmd)()
if options.cmd == 'fetch_glossary':
if options.output is None:
parser.error('--output is required for fetch_glossary')
try:
fetch_glossary(options.output)
except (requests.RequestException, JsonApiException) as error:
# Exceptions may contain signed URLs; report the status without those URLs.
status = (
error.response.status_code
if error.response is not None
else 'network error'
)
parser.exit(1, f'Transifex glossary request failed: {status}.\n')
except (ValueError, KeyError) as error:
parser.exit(1, f'Invalid Transifex glossary export: {error}\n')
else:
eval(options.cmd)()
Loading
Loading