manage_languages.py

# Written for Python 3.6+
# Older versions don't retain insertion order of regular dicts
import argparse
import cmd
import json
import os
import re
from pprint import pprint

INDENT = 2
PRIMARY_LANGUAGE = 'en-US.json'
PRIMARY_FALLBACK_PREFIX = '🇺🇸'  # This is invisible in-game, terminal emulators might render it
LANGUAGE_FOLDER = 'src/main/resources/languages/'
LANGUAGE_FILENAMES = sorted(os.listdir(LANGUAGE_FOLDER), key=lambda x: 'AAA' if x == PRIMARY_LANGUAGE else x)
SOURCE_FOLDER = 'src/'
SOURCE_EXTENSIONS = ('java')


def ppprint(data):
    pprint(data, width=130, sort_dicts=False, compact=True)


class JsonHelpers:
    @staticmethod
    def load(filename: str) -> dict:
        with open(filename, 'r', encoding='utf-8') as file:
            return json.load(file)

    @staticmethod
    def save(filename: str, data: dict) -> None:
        with open(filename, 'w', encoding='utf-8', newline='\n') as file:
            json.dump(data, file, ensure_ascii=False, indent=INDENT)
            file.write('\n')  # json.dump doesn't terminate last line

    @staticmethod
    def flatten(data: dict, prefix='') -> dict:
        output = {}
        for key, value in data.items():
            if isinstance(value, dict):
                for k,v in JsonHelpers.flatten(value, f'{prefix}{key}.').items():
                    output[k] = v
            else:
                output[f'{prefix}{key}'] = value
        return output

    @staticmethod
    def unflatten(data: dict) -> dict:
        output = {}
        def add_key(k: list, value, d: dict):
            if len(k) == 1:
                d[k[0]] = value
            else:
                d[k[0]] = d.get(k[0], {})
                add_key(k[1:], value, d[k[0]])
        for key, value in data.items():
            add_key(key.split('.'), value, output)
        return output

    @staticmethod
    def pprint_keys(keys, indent=4) -> str:
        # Only strip down to one level
        padding = ' ' * indent
        roots = {}
        for key in keys:
            root, _, k = key.rpartition('.')
            roots[root] = roots.get(root, [])
            roots[root].append(k)
        lines = []
        for root, ks in roots.items():
            if len(ks) > 1:
                lines.append(f'{padding}{root}.[{", ".join(ks)}]')
            else:
                lines.append(f'{padding}{root}.{ks[0]}')
        return ',\n'.join(lines)

    @staticmethod
    def deep_clone_and_fill(d1: dict, d2: dict, fallback_prefix=PRIMARY_FALLBACK_PREFIX) -> dict:
        out = {}
        for key, value in d1.items():
            if isinstance(value, dict):
                out[key] = JsonHelpers.deep_clone_and_fill(value, d2.get(key, {}), fallback_prefix)
            else:
                v2 = d2.get(key, value)
                if type(value) == str and v2 == value:
                    out[key] = fallback_prefix + value
                else:
                    out[key] = v2
        return out


class LanguageManager:
    TRANSLATION_KEY = re.compile(r'[Tt]ranslate.*"(\w+\.[\w\.]+)"')
    POTENTIAL_KEY = re.compile(r'"(\w+\.[\w\.]+)"')
    COMMAND_LABEL = re.compile(r'@Command\s*\([\W\w]*?label\s*=\s*"(\w+)"', re.MULTILINE)  # [\W\w] is a cheeky way to match everything including \n

    def __init__(self):
        self.load_jsons()

    def load_jsons(self):
        self.language_jsons = [JsonHelpers.load(LANGUAGE_FOLDER + filename) for filename in LANGUAGE_FILENAMES]
        self.flattened_jsons = [JsonHelpers.flatten(j) for j in self.language_jsons]
        self.update_keys()

    def update_keys(self):
        self.key_sets = [set(j.keys()) for j in self.flattened_jsons]
        self.common_keys = set.intersection(*self.key_sets)
        self.all_keys = set.union(*self.key_sets)
        self.used_keys = self.find_all_used_keys(self.all_keys)
        self.missing_keys = self.used_keys - self.common_keys
        self.unused_keys = self.all_keys - self.used_keys

    def find_all_used_keys(self, expected_keys=[]) -> set:
        # Note that this will only find string literals passed to the translate() or sendTranslatedMessage() methods!
        # String variables passed to them can be checked against expected_keys
        used = set()
        potential = set()
        for root, dirs, files in os.walk(SOURCE_FOLDER):
            for file in files:
                if file.rpartition('.')[-1] in SOURCE_EXTENSIONS:
                    filename = os.path.join(root, file)
                    with open(filename, 'r', encoding='utf-8') as f:
                        data = f.read()  # Loads in entire file at once
                        for k in self.TRANSLATION_KEY.findall(data):
                            used.add(k)
                        for k in self.POTENTIAL_KEY.findall(data):
                            potential.add(k)
                        for label in self.COMMAND_LABEL.findall(data):
                            used.add(f'commands.{label}.description')
        return used | (potential & expected_keys)

    def _lint_report_language(self, lang: str, keys: set, flattened: dict, primary_language_flattened: dict) -> None:
        missing = self.used_keys - keys
        unused = keys - self.used_keys
        identical_keys = set() if (lang == PRIMARY_LANGUAGE) else {key for key in keys if primary_language_flattened.get(key, None) == flattened.get(key)}
        placeholder_keys = {key for key in keys if flattened.get(key).startswith(PRIMARY_FALLBACK_PREFIX)}
        p1 = f'Language {lang} has {len(missing)} missing keys and {len(unused)} unused keys.'
        p2 = 'This is the primary language.' if (lang == PRIMARY_LANGUAGE) else f'{len(identical_keys)} match {PRIMARY_LANGUAGE}, {len(placeholder_keys)} have the placeholder mark.'
        print(f'{p1} {p2}')

        lint_categories = {
            'Missing': missing,
            'Unused': unused,
            f'Matches {PRIMARY_LANGUAGE}': identical_keys,
            'Placeholder': placeholder_keys,
        }
        for name, category in lint_categories.items():
            if len(category) > 0:
                print(name + ':')
                print(JsonHelpers.pprint_keys(sorted(category)))

    def lint_report(self) -> None:
        print(f'There are {len(self.missing_keys)} translation keys in use that are missing from one or more language files.')
        print(f'There are {len(self.unused_keys)} translation keys in language files that are not used.')
        primary_language_flattened = self.flattened_jsons[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)]
        for lang, keys, flattened in zip(LANGUAGE_FILENAMES, self.key_sets, self.flattened_jsons):
            print('')
            self._lint_report_language(lang, keys, flattened, primary_language_flattened)

    def rename_keys(self, key_remappings: dict) -> None:
        # Unfortunately we can't rename keys in-place preserving insertion order, so we have to make new dicts
        for i in range(len(self.flattened_jsons)):
            self.flattened_jsons[i] = {key_remappings.get(k,k):v for k,v in self.flattened_jsons[i].items()}

    def update_secondary_languages(self):
        # Push en_US fallback
        primary_language_json = self.language_jsons[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)]
        for filename, lang in zip(LANGUAGE_FILENAMES, self.language_jsons):
            if filename != PRIMARY_LANGUAGE:
                js = JsonHelpers.deep_clone_and_fill(primary_language_json, lang)
                JsonHelpers.save(LANGUAGE_FOLDER + filename, js)

    def update_all_languages_from_flattened(self):
        for filename, flat in zip(LANGUAGE_FILENAMES, self.flattened_jsons):
            JsonHelpers.save(LANGUAGE_FOLDER + filename, JsonHelpers.unflatten(flat))

    def save_flattened_languages(self, prefix='flat_'):
        for filename, flat in zip(LANGUAGE_FILENAMES, self.flattened_jsons):
            JsonHelpers.save(prefix + filename, flat)


class InteractiveRename(cmd.Cmd):
    intro = 'Welcome to the interactive rename shell.   Type help or ? to list commands.\n'
    prompt = '(rename) '
    file = None

    def __init__(self, language_manager: LanguageManager) -> None:
        super().__init__()
        self.language_manager = language_manager
        self.flat_keys = [key for key in language_manager.flattened_jsons[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)].keys()]
        self.mappings = {}

    def do_add(self, arg):
        '''
        Prepare to rename an existing translation key. Will not actually rename anything until you confirm all your pending changes with 'rename'.
        e.g. a single string:  add commands.execution.argument_error commands.generic.invalid.argument
        e.g. a group:          add commands.enter_dungeon commands.new_enter_dungeon
        '''
        args = arg.split()
        if len(args) < 2:
            self.do_help('add')
            return
        old, new = args[:2]
        if old in self.flat_keys:
            self.mappings[old] = new
        else:
            # Check if we are renaming a higher level
            if not old.endswith('.'):
                old = old + '.'
            results = [key for key in self.flat_keys if key.startswith(old)]
            if len(results) > 0:
                if not new.endswith('.'):
                    new = new + '.'
                new_mappings = {key: key.replace(old, new) for key in results}
                # Ask for confirmation
                print('Will add the following mappings:')
                ppprint(new_mappings)
                print('Add these mappings? [y/N]')
                if self.prompt_yn():
                    for k,v in new_mappings.items():
                        self.mappings[k] = v
            else:
                print('No translation keys matched!')
    
    def complete_add(self, text: str, line: str, begidx: int, endidx: int) -> list:
        if text == '':
            return [k for k in {key.partition('.')[0] for key in self.flat_keys}]
        results = [key for key in self.flat_keys if key.startswith(text)]
        if len(results) > 40:
            # Collapse categories
            if text[-1] != '.':
                text = text + '.'
            level = text.count('.') + 1
            new_results = {'.'.join(key.split('.')[:level]) for key in results}
            return list(new_results)
        return results

    def do_remove(self, arg):
        '''
        Remove a pending rename mapping. Takes the old name of the key, not the new one.
        e.g. a single key:  remove commands.execution.argument_error
        e.g. a group:       remove commands.enter_dungeon
        '''
        old = arg.split()[0]
        if old in self.mappings:
            self.mappings.pop(old)
        else:
            # Check if we are renaming a higher level
            if not old.endswith('.'):
                old = old + '.'
            results = [key for key in self.mappings if key.startswith(old)]
            if len(results) > 0:
                # Ask for confirmation
                print('Will remove the following pending mappings:')
                print(JsonHelpers.pprint_keys(results))
                print('Delete these mappings? [y/N]')
                if self.prompt_yn():
                    for key in results:
                        self.mappings.pop(key)
            else:
                print('No pending rename mappings matched!')
    
    def complete_remove(self, text: str, line: str, begidx: int, endidx: int) -> list:
        return [key for key in self.mappings if key.startswith(text)]

    def do_rename(self, _arg):
        'Applies pending renames and overwrites language jsons.'
        # Ask for confirmation
        print('Will perform the following mappings:')
        ppprint(self.mappings)
        print('Perform and save these rename mappings? [y/N]')
        if self.prompt_yn():
            self.language_manager.rename_keys(self.mappings)
            self.language_manager.update_all_languages_from_flattened()
            print('Renamed keys, closing')
            return True
        else:
            print('Do you instead wish to quit without saving? [yes/N]')
            if self.prompt_yn(True):
                print('Left rename shell without renaming')
                return True

    def prompt_yn(self, strict_yes=False):
        if strict_yes:
            return input('(yes/N) ').lower() == 'yes'
        return input('(y/N) ').lower()[0] == 'y'


def main(args: argparse.Namespace):
    # print(args)
    language_manager = LanguageManager()
    errors = None
    if args.lint_report:
        language_manager.lint_report()
        missing = language_manager.used_keys - language_manager.key_sets[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)]
        if len(missing) > 0:
            errors = f'[ERROR] {len(missing)} keys missing from primary language json!\n{JsonHelpers.pprint_keys(missing)}'
    if prefix := args.save_flattened:
        language_manager.save_flattened_languages(prefix)
    if args.update:
        print('Updating secondary languages')
        language_manager.update_secondary_languages()
    if args.interactive_rename:
        language_manager.load_jsons()  # Previous actions may have changed them on-disk
        try:
            InteractiveRename(language_manager).cmdloop()
        except KeyboardInterrupt:
            print('Left rename shell without renaming')
    if errors:
        print(errors)
        exit(1)



if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Manage Grasscutter's language json files.")
    parser.add_argument('-u', '--update', action='store_true',
        help=f'Update secondary language files to conform to the layout of the primary language file ({PRIMARY_LANGUAGE}) and contain any new keys from it.')
    parser.add_argument('-l', '--lint-report', action='store_true',
        help='Prints a lint report, listing unused, missing, and untranslated keys among all language jsons.')
    parser.add_argument('-f', '--save-flattened', const='./flat_', metavar='prefix', nargs='?',
        help='Save copies of all the language jsons in a flattened key form.')
    parser.add_argument('-i', '--interactive-rename', action='store_true',
        help='Enter interactive rename mode, in which you can specify keys in flattened form to be renamed.')
    args = parser.parse_args()
    main(args)