Android에서 사용하는 다국어 개선하기

from xml.etree import ElementTree

class StringRes:
    def __init__(self, path):
        self.path = path
        self.res = self._get_res()
        self.root = self.res.getroot()

    def _get_res(self):
        return ElementTree.parse(self.path)

    def parse(self):
        return {child.attrib["name"]: child.text for child in self.root.iter("string")}

    def remove(self, remove_keys):
        for key in remove_keys:
            element = self._find_element(key)
            if element is not None:
                self.root.remove(element)

    def replace(self, replace_dict):
        for key, value in replace_dict.items():
            element = self._find_element(key)
            if element is not None:
                element.text = replace_dict[key]

    def insert(self, insert_dict):
        for key, value in insert_dict.items():
            if self._find_element(key) is None:
                element = ElementTree.Element("string")
                element.set("name", key)
                element.text = value
                self.root.append(element)

    def _find_element(self, key):
        key_path = './/string[@name="%s"]' % key
        return self.root.find(key_path)

    def save(self, path=None):
        if path is None:
            path = self.path
        self._indent()
        self.res.write(path, encoding="UTF-8", xml_declaration=True)

    def _indent(self):
        if not len(self.root):
            return
        level = 0
        space = "\t"
        indentations = ["\n" + level * space]

        def _indent_children(element, level):
            child_level = level + 1
            try:
                child_indentation = indentations[child_level]
            except IndexError:
                child_indentation = indentations[level] + space
                indentations.append(child_indentation)

            if not element.text or not element.text.strip():
                element.text = child_indentation

            for child in element:
                if len(child):
                    _indent_children(child, child_level)
                if not child.tail or not child.tail.strip():
                    child.tail = child_indentation

            if not child.tail.strip():
                child.tail = indentations[level]

        _indent_children(self.root, 0)

    def log(self):
        ElementTree.dump(self.root)

import os
import mmap
from glob import iglob

def search_modules(modules):
    for module in modules:
        find_in_kotlin(module)
        find_in_java(module)
        find_in_resource(module)

def find_in_kotlin(module):
    path = get_root_path() + module + _SRC_MAIN_PATH + "java/**/*.kt"
    for file in iglob(path, recursive=True):
        find_usage_in_file(file)

def find_in_java(module):
    path = get_root_path() + module + _SRC_MAIN_PATH + "java/**/*.java"
    for file in iglob(path, recursive=True):
        find_usage_in_file(file)

def find_in_resource(module):
    path = get_root_path() + module + _SRC_MAIN_PATH + "res*/**/*.xml"
    for file in iglob(path, recursive=True):
        if not os.path.basename(file).startswith("string"):
            find_usage_in_file(file)

def find_usage_in_file(file_path):
    global _all_keys
    with open(file_path, 'rb', 0) as f:
        s = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
        used_keys = {key for key in _all_keys if s.find(bytes(key, 'utf-8')) != -1}
        print("Usage in " + file_path)
        print(used_keys)

def compare_dict(lokalise_dict, strings_dict):
    lokalise_keys = set(lokalise_dict.keys())
    strings_keys = set(strings_dict.keys())

    already_in_strings = strings_keys.intersection(lokalise_keys)
    modified = {key: lokalise_dict[key] for key in already_in_strings if strings_dict[key] != lokalise_dict[key]}
    add_to_strings = {key: lokalise_dict[key]
                      for key in lokalise_keys.difference(already_in_strings).intersection(_strings_local_keys)}
    remove_from_strings = {key: strings_dict[key] for key in strings_keys.difference(already_in_strings)}
    return modified, add_to_strings, remove_from_strings

def sync_with_lokalise():
    def sync_values_with_lokalise(values):
        lokalise_res = StringRes(LOKALISE_ROOT_PATH + values + STRINGS_XML)
        strings_res = StringRes(STRING_ROOT_PATH + values + STRINGS_XML)
        strings_local_res = StringRes(STRING_ROOT_PATH + values + STRINGS_LOCAL_XML)
        modified, add_to_strings, remove_from_strings = compare_dict(lokalise_res.parse(), strings_res.parse())

        strings_res.remove(remove_from_strings)
        strings_res.insert(add_to_strings)
        strings_res.replace(modified)
        strings_res.save()

        strings_local_res.insert(remove_from_strings)
        strings_local_res.remove(add_to_strings)
        if values != VALUES_EN:
            strings_local_res.remove(check_lokalise_untranslated(strings_local_res.parse()))
        strings_local_res.save()

    for values in ALL_VALUES:
        sync_values_with_lokalise(values)

import collections
from pprint import pprint

def find_duplicate():
    all_duplicate = set()
    for value in ALL_VALUES:
        duplicate = find_duplicate_in_value(value)
        if all_duplicate:
            all_duplicate = all_duplicate.intersection(duplicate)
        else:
            all_duplicate = all_duplicate.union(duplicate)
    pprint(all_duplicate)

def find_duplicate_in_value(value):
    all_dicts = get_all_strings(value)
    value_to_key = collections.defaultdict(list)
    for key, value in all_dicts.items():
        value_to_key[value].append(key)

    return {frozenset(value) for key, value in value_to_key.items() if len(value) > 1}

...

def find_unused_in_file(file_path):
    global _all_keys, _all_used_keys
    with open(file_path, 'rb', 0) as f:
        s = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
        used_keys = {key for key in _all_keys if s.find(bytes(key, 'utf-8')) != -1}
        _all_used_keys = _all_used_keys.union(used_keys)

...
unused_keys = _all_keys - _all_used_keys
print("\\n".join(unused_keys))

pipeline {
    agent any

    environment {
        BRANCH = ''
        DOWNLOAD_LOKALISE = ''
    }

    stages {
        stage('Pull Latest') {
            steps {
                script {
                    BRANCH = withoutOrigin(env.GIT_BRANCH)
                }
                withEnv(["""BRANCH=$BRANCH"""]) {
                    sh 'python3.7 tools/checkout_branch.py'
                }
                sh "git pull origin $BRANCH"
            }
        }

        stage('Download Lokalise') {
            steps {
                sh 'python -m pip install requests --user'
                script {
                    DOWNLOAD_LOKALISE = """${sh(script: 'python tools/string/download_lokalise.py', returnStatus: true)}"""
                }
            }
        }

        stage('Sync with Strings Local') {
            when {
                expression { DOWNLOAD_LOKALISE == '0' }
            }
            steps {
                sh 'python tools/string/sync_strings_local.py'
            }
        }

        stage('Check and Commit'){
            when {
                expression { DOWNLOAD_LOKALISE == '0' }
            }
            steps {
                withEnv(["""BRANCH=$BRANCH"""]) {
                    script {
                        def result = sh(script: 'python3.7 tools/string/check_and_commit.py', returnStdout: true).split('\\n').last()
                    }
                }
            }
        }
    }
}

def withoutOrigin(branch) {
    return branch.startsWith('origin/') ? branch - 'origin/' : branch
}

def get_lokalise_download_url():
    url = _LOKALISE_FILES_DOWNLOAD_URL % _LOKALISE_PROJECT_ID
    params = {
        "filter_langs": ["en", "ja", "ko"],
        "include_tags": ["common", "core", "gallery", "voiceover_android"],
        "exclude_tags": ["unused_android"],
        "format": "xml",
        "original_filenames": False,
        "all_platforms": False,
        "indentation": "tab",
        "add_newline_eof": True,
        "export_sort": "first_added",
        "export_empty_as": "skip",  # 해당 언어 번역이 없을 경우 skip
        "bundle_description": "android_sdk",
        "bundle_structure": "values-%LANG_ISO%/strings.%FORMAT%"
    }
    headers = {
        "Accept": "application/json",
        "Content-Type": "application/json",
        "X-Api-Token": _LOKALISE_API_TOKEN
    }
    json_response = json.loads(requests.post(url, json=params, headers=headers).text)
    if 'bundle_url' in json_response:
        return json_response['bundle_url']
    else:
        return None

def sync_strings_with_local():
    def sync_values_strings_with_local(values):
        strings_res = StringRes(get_jenkins_string_root_path() + values + STRINGS_XML)
        strings_local_res = StringRes(get_jenkins_string_root_path() + values + STRINGS_LOCAL_XML)
        remove_from_local = compare_strings_with_local(strings_res.parse(), strings_local_res.parse())

        strings_local_res.remove(remove_from_local)
        strings_local_res.save()

    for values in ALL_VALUES:
        sync_values_strings_with_local(values)

def compare_strings_with_local(strings_dict, strings_local_dict):
    strings_keys = set(strings_dict.keys())
    strings_local_keys = set(strings_local_dict.keys())

    remove_from_local = strings_keys.intersection(strings_local_keys)
    remove_from_local.update(_en_strings_keys.intersection(strings_local_keys))
    return remove_from_local

import subprocess as cmd
import os

def check_changed():
    cmd.run("git add .", check=True, shell=True)
    proc = cmd.run("git diff --quiet HEAD", capture_output=True, shell=True)
    return bool(proc.returncode)

if __name__ == '__main__':
    branch = os.environ["BRANCH"]
    if check_changed():
        cmd.run("git commit -m 'updated strings.'", check=True, shell=True)
        cmd.run("git push origin %s" % branch, check=True, shell=True)