Skip to content

Commit b5db589

Browse files
kazanzhykazanzhy
committed
Use transifex-python lib with v3 API. Add stat files
1 parent ffe77bf commit b5db589

File tree

6 files changed

+656
-220
lines changed

6 files changed

+656
-220
lines changed

Diff for: ‎.github/scripts/manage_translation.py

+86-200
Original file line numberDiff line numberDiff line change
@@ -2,215 +2,101 @@
22
#
33
# This python file contains utility scripts to manage Python docs Ukrainian translation.
44
# It has to be run inside the python-docs-uk git root directory.
5-
#
6-
# Inspired by django-docs-translations script by claudep.
7-
#
8-
# The following commands are available:
9-
#
10-
# * fetch: fetch translations from transifex.com and strip source lines from the
11-
# files.
12-
# * recreate_readme: recreate readme to update translation progress.
13-
# * regenerate_tx_config: recreate configuration for all resources.
145

15-
from argparse import ArgumentParser
16-
from collections import Counter
176
import os
18-
from re import match
19-
from subprocess import call, run
20-
import sys
7+
import re
8+
from argparse import ArgumentParser
9+
from pathlib import Path
2110

22-
LANGUAGE = 'uk'
23-
RESOURCE_NAME_MAP = {'glossary_': 'glossary'}
24-
TX_ORGANISATION = 'python-doc'
25-
TX_PROJECT = 'python-newest'
26-
GH_ORGANISATION = 'python'
27-
GH_PROJECT = 'python-docs-uk'
28-
29-
def fetch():
30-
"""
31-
Fetch translations from Transifex, remove source lines.
32-
"""
33-
if call("tx --version", shell=True) != 0:
34-
sys.stderr.write("The Transifex client app is required (https://developers.transifex.com/docs/cli).\n")
35-
exit(1)
36-
lang = LANGUAGE
37-
pull_returncode = call(f'tx pull -l {lang} --minimum-perc=1 --force --skip', shell=True)
38-
if pull_returncode != 0:
39-
exit(pull_returncode)
40-
for root, _, po_files in os.walk('../..'):
41-
for po_file in po_files:
42-
if not po_file.endswith(".po"):
43-
continue
44-
po_path = os.path.join(root, po_file)
45-
call(f'msgcat --no-location -o {po_path} {po_path}', shell=True)
46-
47-
48-
def recreate_tx_config():
49-
"""
50-
Regenerate Transifex client config for all resources.
51-
"""
52-
resources = _get_resources()
53-
with open('.tx/config', 'w') as config:
54-
config.writelines(('[main]\n', 'host = https://www.transifex.com\n',))
55-
for resource in resources:
56-
slug = resource['slug']
57-
name = RESOURCE_NAME_MAP.get(slug, slug)
58-
if slug == '0':
59-
continue
60-
elif '--' in slug:
61-
directory, file_name = name.split('--')
62-
if match(r'\d+_\d+', file_name):
63-
file_name = file_name.replace('_', '.')
64-
config.writelines(
65-
(
66-
'\n',
67-
f'[{TX_PROJECT}.{slug}]\n',
68-
f'trans.{LANGUAGE} = {directory}/{file_name}.po\n',
69-
'type = PO\n',
70-
'source_lang = en\n',
71-
)
72-
)
73-
else:
74-
config.writelines(
75-
(
76-
'\n',
77-
f'[{TX_PROJECT}.{slug}]\n',
78-
f'trans.{LANGUAGE} = {name}.po\n',
79-
'type = PO\n',
80-
'source_lang = en\n',
81-
)
82-
)
83-
84-
85-
def _get_resources():
86-
from requests import get
87-
88-
resources = []
89-
offset = 0
90-
if os.path.exists('.tx/api-key'):
91-
with open('.tx/api-key') as f:
92-
transifex_api_key = f.read()
93-
else:
94-
transifex_api_key = os.getenv('TX_TOKEN')
95-
while True:
96-
response = get(
97-
f'https://api.transifex.com/organizations/{TX_ORGANISATION}/projects/{TX_PROJECT}/resources/',
98-
params={'language_code': LANGUAGE, 'offset': offset},
99-
auth=('api', transifex_api_key),
100-
)
101-
response.raise_for_status()
102-
response_list = response.json()
103-
resources.extend(response_list)
104-
if len(response_list) < 100:
105-
break
106-
offset += len(response_list)
107-
return resources
108-
109-
110-
def _get_unique_translators():
111-
process = run(
112-
['grep', '-ohP', r'(?<=^# )(.+)(?=, \d+$)', '-r', '.'],
113-
capture_output=True,
114-
text=True,
115-
)
116-
translators = [match('(.*)( <.*>)?', t).group(1) for t in process.stdout.splitlines()]
117-
unique_translators = Counter(translators)
118-
return unique_translators
119-
120-
121-
def recreate_readme():
122-
def language_switcher(entry):
123-
return (
124-
entry['name'].startswith('bugs')
125-
or entry['name'].startswith('tutorial')
126-
or entry['name'].startswith('library--functions')
127-
)
128-
129-
def average(averages, weights):
130-
return sum([a * w for a, w in zip(averages, weights)]) / sum(weights)
131-
132-
resources = _get_resources()
133-
filtered = list(filter(language_switcher, resources))
134-
average_list = [e['stats']['translated']['percentage'] for e in filtered]
135-
weights_list = [e['wordcount'] for e in filtered]
136-
137-
language_switcher_status = average(average_list, weights=weights_list) * 100
138-
unique_translators = _get_unique_translators()
139-
number_of_translators = len(unique_translators)
140-
141-
with open('README.md', 'w') as file:
142-
file.write(
143-
f'''\
144-
Український переклад документації Python
145-
========================================
146-
![build](https://github.com/{GH_ORGANISATION}/{GH_PROJECT}/workflows/.github/workflows/update-and-build.yml/badge.svg)
147-
![{language_switcher_status:.2f}% прогрес перекладу](https://img.shields.io/badge/прогрес_перекладу-{language_switcher_status:.2f}%25-0.svg)
148-
![хід перекладу всієї документації](https://img.shields.io/badge/dynamic/json.svg?label=всього&query=$.{LANGUAGE}&url=http://gce.zhsj.me/python/newest)
149-
![{number_of_translators} перекладачів](https://img.shields.io/badge/перекладачів-{number_of_translators}-0.svg)
150-
151-
Якщо ви знайшли помилку або маєте пропозицію,
152-
[додати issue](https://github.com/{GH_ORGANISATION}/{GH_PROJECT}/issues) у цьому проекті або запропонуйте зміни:
153-
154-
* Зареєструйтесь на платформі [Transifex](https://www.transifex.com/)
155-
* Перейдіть на сторінку [документації Python](https://www.transifex.com/{TX_ORGANISATION}/{TX_PROJECT}/).
156-
* Натисніть кнопку „Join Team”, оберіть українську (uk) мову та натисніть „Join” щоб приєднатися до команди.
157-
* Приєднавшись до команди, виберіть ресурс, що хочете виправити/оновити.
158-
159-
Додаткову інформацію про використання Transifex дивіться [в документації](https://docs.transifex.com/getting-started-1/translators).
160-
161-
**Прогрес перекладу**
162-
163-
![прогрес перекладу](language-switcher-progress.svg)
164-
165-
Українська мова з’явиться в меню вибору мови docs.python.org, [коли будуть повністю перекладені](https://www.python.org/dev/peps/pep-0545/#add-translation-to-the-language-switcher):
166-
* `bugs`,
167-
* всі ресурси в каталозі `tutorial`,
168-
* `library/functions`.
169-
170-
**Як переглянути останню збірку документації?**
171-
172-
Завантажте останню створену документацію зі списку артефактів в останній дії GitHub (вкладка Actions).
173-
Переклади завантажуються з Transifex до цього репозиторію приблизно кожні півгодини.
174-
Документація на python.org оновлюється приблизно раз на день.
175-
176-
**Канали зв'язку**
177-
178-
* [Telegram-чат перекладачів](https://t.me/+dXwqHZ0KPKYyNDc6)
179-
* [Python translations working group](https://mail.python.org/mailman3/lists/translation.python.org/)
180-
* [Python Documentation Special Interest Group](https://www.python.org/community/sigs/current/doc-sig/)
181-
182-
**Ліцензія**
183-
184-
Запрошуючи вас до спільного створення проекту на платформі Transifex, ми пропонуємо договір на передачу ваших перекладів
185-
Python Software Foundation [по ліцензії CC0](https://creativecommons.org/publicdomain/zero/1.0/deed.uk).
186-
Натомість ви побачите, що ви є перекладачем тієї частини, яку ви переклали.
187-
Ви висловлюєте свою згоду з цією угодою, надаючи свою роботу для включення в документацію.
188-
189-
**Оновлення локального перекладу**
190-
191-
* `.github/scripts/manage_translation.py recreate_tx_config`
192-
* `.github/scripts/manage_translation.py fetch`
193-
* `.github/scripts/manage_translation.py recreate_readme`
194-
195-
**Подяка**
196-
* Maciej Olko - Polish team
197-
* Julien Palard - French team
198-
* Tomo Cocoa - Japanese team
11+
from transifex.api import transifex_api
19912

200-
**Внесок спільноти**
20113

202-
| Перекладач | Кількість документів |
203-
|:----------------|:--------------------:|
204-
'''
205-
)
14+
transifex_api.setup(auth=os.getenv('TX_TOKEN'))
20615

207-
file.writelines(
208-
(f'|{t}|{c}|\n' for (t, c) in unique_translators.most_common())
209-
)
16+
RESOURCE_NAME_MAP = {'glossary_': 'glossary'}
17+
18+
ORGANISATION_ID = 'o:python-doc'
19+
PROJECT_ID = 'o:python-doc:p:python-newest'
20+
LANGUAGE_ID = 'l:uk'
21+
ORGANISATION = transifex_api.Organization.get(id=ORGANISATION_ID)
22+
PROJECT = transifex_api.Project.get(id=PROJECT_ID)
23+
LANGUAGE = transifex_api.Language.get(id=LANGUAGE_ID)
24+
25+
26+
def _slug_to_file_path(slug: str) -> Path:
27+
"""Set of rules how to transform slug to translation file path"""
28+
file_path = RESOURCE_NAME_MAP.get(slug, slug) # Legacy slug to file mapping
29+
file_path = file_path.replace('--', '/')
30+
if re.match(r'\d+_\d+', file_path):
31+
file_path = file_path.replace('_', '.')
32+
file_path = file_path + '.po'
33+
return Path(file_path)
34+
35+
36+
def recreate_config() -> None:
37+
"""Regenerate Transifex client config for all resources."""
38+
resources = transifex_api.Resource.filter(project=PROJECT).all()
39+
with open('.tx/config', 'w') as fo:
40+
fo.writelines(('[main]\n', 'host = https://api.transifex.com\n',))
41+
for resource in resources:
42+
path = _slug_to_file_path(resource.slug)
43+
fo.writelines((
44+
'\n',
45+
f'[{resource.id}]\n',
46+
f'file_filter = {path}\n',
47+
'type = PO\n',
48+
'source_lang = en\n',
49+
))
50+
51+
52+
def recreate_resource_stats() -> None:
53+
"""Create resource stats."""
54+
stats = transifex_api.ResourceLanguageStats.filter(project=PROJECT, language=LANGUAGE).all()
55+
with open('RESOURCE.md', 'w') as fo:
56+
fo.writelines(('| Файл | Перекладено | Переглянуто | Вичитано |\n', '|:-----|:-----|:-----|:-----|\n'))
57+
for stat in stats:
58+
file_name = _slug_to_file_path(stat.id.split(':')[5])
59+
translated_pct = round(100 * stat.attributes['translated_words'] / stat.attributes['total_words'], 1)
60+
reviewed_pct = round(100 * stat.attributes['reviewed_words'] / stat.attributes['total_words'], 1)
61+
proofread_pct = round(100 * stat.attributes['proofread_words'] / stat.attributes['total_words'], 1)
62+
fo.writelines(f'| {file_name} | {translated_pct} % | {reviewed_pct} % | {proofread_pct} % |\n')
63+
64+
65+
def recreate_team_stats() -> None:
66+
"""Create contributor stats"""
67+
members = transifex_api.TeamMembership.filter(organization=ORGANISATION, language=LANGUAGE).all()
68+
69+
users = {member.user.id: member.attributes['role'] for member in members}
70+
translators = dict.fromkeys(users.keys(), 0)
71+
reviewers = dict.fromkeys(users.keys(), 0)
72+
proofreaders = dict.fromkeys(users.keys(), 0)
73+
74+
resources = transifex_api.Resource.filter(project=PROJECT).all()
75+
for resource in resources:
76+
translations = transifex_api.ResourceTranslation.filter(resource=resource, language=LANGUAGE).all()
77+
for translation in translations:
78+
if translation.relationships['translator']:
79+
translators[translation.relationships['translator']['data']['id']] += 1
80+
if translation.relationships['reviewer']:
81+
reviewers[translation.relationships['reviewer']['data']['id']] += 1
82+
if translation.relationships['proofreader']:
83+
proofreaders[translation.relationships['proofreader']['data']['id']] += 1
84+
85+
with open('TEAM.md', 'w') as fo:
86+
fo.writelines(('| | Роль | Переклав | Переглянув | Вичитав |\n', '|:---|:---|:---|:---|:---|\n',))
87+
for user, role in users.items():
88+
fo.writelines(f"| {user} | {role} | {translators[user]} | {reviewers[user]} | {proofreaders[user]} |\n")
89+
90+
91+
def fetch_translations():
92+
"""Fetch translations from Transifex, remove source lines."""
93+
pull_return_code = os.system(f'tx pull -l uk --force --skip')
94+
if pull_return_code != 0:
95+
exit(pull_return_code)
21096

21197

21298
if __name__ == "__main__":
213-
RUNNABLE_SCRIPTS = ('fetch', 'recreate_tx_config', 'recreate_readme')
99+
RUNNABLE_SCRIPTS = ('recreate_config', 'recreate_resource_stats', 'recreate_team_stats', 'fetch_translations')
214100

215101
parser = ArgumentParser()
216102
parser.add_argument('cmd', nargs=1, choices=RUNNABLE_SCRIPTS)

Diff for: ‎.github/workflows/update-and-build.yml

+7-11
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
on:
22
schedule:
3-
- cron: '0 0 * * *'
3+
- cron: '0 6 * * *'
44
push:
55
branches: ['main']
6+
workflow_dispatch:
67

78
jobs:
89
update-translation:
@@ -17,24 +18,19 @@ jobs:
1718
- uses: actions/setup-python@master
1819
with:
1920
python-version: 3
20-
- run: sudo apt-get install -y gettext
2121
- run: curl -o- https://raw.githubusercontent.com/transifex/cli/master/install.sh | bash
2222
working-directory: /usr/local/bin
23-
- run: pip install requests
23+
- run: pip install transifex-python
2424
- uses: actions/checkout@master
2525
with:
2626
ref: ${{ matrix.version }}
27-
- run: .github/scripts/manage_translation.py recreate_tx_config
27+
- run: .github/scripts/manage_translation.py recreate_config
2828
env:
2929
TX_TOKEN: ${{ secrets.TX_TOKEN }}
30-
- run: .github/scripts/manage_translation.py fetch
30+
- run: .github/scripts/manage_translation.py fetch_translations
3131
env:
3232
TX_TOKEN: ${{ secrets.TX_TOKEN }}
33-
- run: .github/scripts/manage_translation.py recreate_readme
34-
env:
35-
TX_TOKEN: ${{ secrets.TX_TOKEN }}
36-
- run: git config --local user.email '[email protected]'
37-
name: Run git config --local user.email '…'
33+
- run: git config --local user.email [email protected]
3834
- run: git config --local user.name "GitHub Action's update-translation job"
3935
- run: git add .
4036
- run: git commit -m 'Update translation from Transifex' || true
@@ -56,7 +52,7 @@ jobs:
5652
steps:
5753
- uses: actions/setup-python@master
5854
with:
59-
python-version: '3.10'
55+
python-version: 3
6056
- uses: actions/checkout@master
6157
with:
6258
repository: python/cpython

Diff for: ‎.tx/config

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
[main]
2-
host = https://www.transifex.com
2+
host = https://api.transifex.com

0 commit comments

Comments
 (0)