#!/bin/python3
import argparse
import json
import os
import sys
# This allows for importing from the localization and util directories NOTE: Auto importing tools will also prepend the import paths with "tools." this will not work and needs to be removed from import paths
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from util.time import ExecutionTimer;
timer = ExecutionTimer()
from dynamicVariables import (
extractVariablesFromDict,
identifyLocaleDynamicVariableDifferences,
prettyPrintIssuesTable,
identifyAndPrintOldDynamicVariables, extractFormattingTags,
)
from localization.localeTypes import generateLocalesType
from util.logger import console
from util.fileUtils import createMappedJsonFileDictionary, writeFile
# These string keys are ignored for formatting tag checks
ignored_strings_formatting = {
"pl": [
# disappearingMessagesTurnedOffYouGroup in pl only has one bold word as the word combines both bold words
"disappearingMessagesTurnedOffYouGroup"]
}
# If the --throw-error-on-missing flag is passed, the script will exit with an error if there are any missing keys or dynamic variables
# This is useful for CI/CD pipelines to ensure that all translations are consistent
parser = argparse.ArgumentParser(description="Generate locale files")
parser.add_argument(
"--error-on-problems",
action="store_true",
help="Exit with an error if there are any missing keys or dynamic variables",
)
parser.add_argument(
"--error-old-dynamic-variables",
action="store_true",
help="Exit with an error if there are any old dynamic variables",
)
parser.add_argument(
"--print-problems",
action="store_true",
help="Print the problems table",
)
parser.add_argument(
"--print-problem-strings",
action="store_true",
help="Print the problem strings and which locales they are in",
)
parser.add_argument(
"--print-problem-formatting-tag-strings",
action="store_true",
help="Print the problem strings and which locales they are in",
)
parser.add_argument(
"--write-problems", action="store_true", help="Write the problems to a file"
)
parser.add_argument(
"--problems-file",
default="./tools/localization/output/problems.json",
help="The file to write the problems to",
)
parser.add_argument(
"--print-old-dynamic-variables",
action="store_true",
help="The file to write the problems to",
)
parser.add_argument("--en-only", action="store_true", help="Only check the en locale")
parser.add_argument("--debug", action="store_true", help="Enable debug mode")
parser.add_argument(
"--dict-dir",
type=str,
default="./_locales"
)
parser.add_argument(
"--dict-file-name",
type=str,
default="messages.json",
)
parser.add_argument(
"--en-file-path",
type=str,
default="./_locales/en/messages.json",
)
parser.add_argument(
"--generate-types",
action="store_true",
help="Generate the types file",
)
args = parser.parse_args()
if args.debug:
console.enableDebug()
GENERATE_TYPES = args.generate_types
OUTPUT_DIR = "./ts/localization"
EN_FILE = args.en_file_path
INPUT_DIR = args.dict_dir
# Create a dictionary that maps locale names to their corresponding JSON file data
locales, localeFiles = createMappedJsonFileDictionary(INPUT_DIR, args.dict_file_name)
if args.en_only:
locales = {"en": locales["en"]}
# Generate the locales type and write it to a file
if GENERATE_TYPES:
generateTypesOutputMessage = generateLocalesType(locales["en"])
console.info(generateTypesOutputMessage)
localeVariables = dict()
localeVariablesOld = dict()
locale_b_tags = dict()
locale_br_tags = dict()
locale_span_tags = dict()
locale_disallowed_tags = dict()
locale_improper_tags = dict()
# Extract the dynamic variables from each locale and store them in a dictionary
for locale, data in locales.items():
console.debug(f"Extracting dynamic variables for {locale}")
(
localeVariables[locale],
localeVariablesOld[locale],
) = extractVariablesFromDict(data)
(
locale_b_tags[locale],
locale_br_tags[locale],
locale_span_tags[locale],
locale_disallowed_tags[locale],
locale_improper_tags[locale],
) = extractFormattingTags(data)
problems = identifyLocaleDynamicVariableDifferences(localeVariables, locale_b_tags,
locale_br_tags,
locale_span_tags, locale_disallowed_tags, locale_improper_tags)
found_old_dynamic_variables = identifyAndPrintOldDynamicVariables(
localeVariablesOld, args.print_old_dynamic_variables
)
# Wrapping up the script and printing out the results
number_of_tag_problems = 0
if problems:
message = "There are issues with the locales."
if args.print_problem_strings:
string_to_locales = {}
for locale, locale_problems in problems.items():
if "additional_variables" in locale_problems:
for problem_string in locale_problems["additional_variables"].keys():
if problem_string not in string_to_locales:
string_to_locales[problem_string] = [locale]
else:
string_to_locales[problem_string].append(locale)
if "missing_variables" in locale_problems:
for problem_string in locale_problems["missing_variables"].keys():
if problem_string not in string_to_locales:
string_to_locales[problem_string] = [locale]
else:
string_to_locales[problem_string].append(locale)
if "missing_br_tags" in locale_problems:
for problem_string, tag_issues in locale_problems["missing_br_tags"].items():
if tag_issues > 0:
if problem_string not in string_to_locales:
string_to_locales[problem_string] = [locale]
else:
string_to_locales[problem_string].append(locale)
if "missing_b_tags" in locale_problems:
for problem_string, tag_issues in locale_problems["missing_b_tags"].items():
if tag_issues > 0:
if problem_string not in string_to_locales:
string_to_locales[problem_string] = [locale]
else:
string_to_locales[problem_string].append(locale)
if "missing_span_tags" in locale_problems:
for problem_string, tag_issues in locale_problems["missing_span_tags"].items():
if tag_issues > 0:
if problem_string not in string_to_locales:
string_to_locales[problem_string] = [locale]
else:
string_to_locales[problem_string].append(locale)
if "disallowed_tags" in locale_problems:
for problem_string, tag_issues in locale_problems["disallowed_tags"].items():
if tag_issues > 0:
if problem_string not in string_to_locales:
string_to_locales[problem_string] = [locale]
else:
string_to_locales[problem_string].append(locale)
if "improper_tags" in locale_problems:
for problem_string, tag_issues in locale_problems["improper_tags"].items():
if tag_issues > 0:
if problem_string not in string_to_locales:
string_to_locales[problem_string] = [locale]
else:
string_to_locales[problem_string].append(locale)
console.debug(f"Problem strings: {json.dumps(string_to_locales, indent=2)}")
message += " See above for problem strings and which locales they are in."
if args.print_problem_formatting_tag_strings:
locales_to_strings = {}
for locale, locale_problems in problems.items():
locale_missing_br_tags = set()
locale_missing_b_tags = set()
locale_missing_span_tags = set()
locale_disallowed_tags = set()
locale_improper_tags = set()
if "missing_br_tags" in locale_problems:
for problem_string, tag_issues in locale_problems["missing_br_tags"].items():
if tag_issues > 0:
locale_missing_br_tags.add(problem_string)
if "missing_b_tags" in locale_problems:
for problem_string, tag_issues in locale_problems["missing_b_tags"].items():
if tag_issues > 0:
locale_missing_b_tags.add(problem_string)
if "missing_span_tags" in locale_problems:
for problem_string, tag_issues in locale_problems["missing_span_tags"].items():
if tag_issues > 0:
locale_missing_span_tags.add(problem_string)
if "disallowed_tags" in locale_problems:
for problem_string, tag_issues in locale_problems["disallowed_tags"].items():
if tag_issues > 0:
locale_disallowed_tags.add(problem_string)
if "improper_tags" in locale_problems:
for problem_string, tag_issues in locale_problems["improper_tags"].items():
if tag_issues > 0:
locale_improper_tags.add(problem_string)
locales_to_strings[locale] = {
"br": list(locale_missing_br_tags),
"b": list(locale_missing_b_tags),
"span": list(locale_missing_span_tags),
"disallowed_tags": list(locale_disallowed_tags),
"improper_tags": list(locale_improper_tags),
}
if locales_to_strings[locale]["br"] == []:
del locales_to_strings[locale]["br"]
if locales_to_strings[locale]["b"] == []:
del locales_to_strings[locale]["b"]
if locales_to_strings[locale]["span"] == []:
del locales_to_strings[locale]["span"]
if locales_to_strings[locale]["disallowed_tags"] == []:
del locales_to_strings[locale]["disallowed_tags"]
if locales_to_strings[locale]["improper_tags"] == []:
del locales_to_strings[locale]["improper_tags"]
console.info(f"Problem strings: {json.dumps(locales_to_strings, indent=2)}")
message += " See above for problem strings and which locales they are in."
for locale, locale_strings in locales_to_strings.items():
printed_locale = False
printed_problem_strings = set()
for tag_type, tag_strings in locale_strings.items():
if tag_strings:
if locale in ignored_strings_formatting and tag_strings == ignored_strings_formatting[locale]:
continue
if not printed_locale:
print(f"{locale} - [Link Here](https://crowdin.com/editor/session-crossplatform-strings/300/en-{locale})")
printed_locale = True
for tag_string in tag_strings:
if tag_string not in printed_problem_strings:
printed_problem_strings.add(tag_string)
number_of_tag_problems += 1
print(
f"- [{tag_string}](https://crowdin.com/editor/session-crossplatform-strings/300/en-{locale}?view=comfortable&filter=basic&value=3#q={tag_string})")
print(f"Total Problems: {number_of_tag_problems}")
if args.print_problems:
prettyPrintIssuesTable(problems)
message += " See above for details."
if args.write_problems:
writeFile(args.problems_file, json.dumps(problems, indent=2))
console.info(f"Problems written to {args.problems_file}")
message += f" Problems written to {args.problems_file}"
if not args.print_problems and not args.write_problems:
message += " Run the script with --print-problems or --write-problems to see the problems."
console.warn(message)
if found_old_dynamic_variables:
warning_message = (
"Old dynamic variables were found in the locales. Please update the locales to use the new dynamic variables. "
)
if args.print_old_dynamic_variables:
if args.print_problems:
warning_message += "See above for details (before the problems table)."
else:
warning_message += "See above for details."
else:
warning_message += "Run the script with --print-old-dynamic-variables to see the old dynamic variables."
console.warn(warning_message)
console.debug("Locales generation complete")
timer.stop()
if args.error_on_problems:
missing_keys_all = 0
additional_keys_all = 0
missing_variables_all = 0
additional_variables_all = 0
for locale_name, locale_issues in problems.items():
if locale_name == "en":
continue
missing_keys_all += len(locale_issues.get("missing_keys", []))
additional_keys_all += len(locale_issues.get("additional_keys", []))
missing_variables_all += sum(
len(v) for v in locale_issues.get("missing_variables", {}).values()
)
additional_variables_all += sum(
len(v) for v in locale_issues.get("additional_variables", {}).values()
)
EXIT_CODE = 0
if missing_keys_all > 0:
console.log(f"Missing keys: {missing_keys_all}")
if additional_keys_all > 0:
console.log(f"Additional keys: {additional_keys_all}")
if missing_variables_all > 0:
console.log(f"Missing variables: {missing_variables_all}")
EXIT_CODE = 1
if additional_variables_all > 0:
console.log(f"Additional variables: {additional_variables_all}")
EXIT_CODE = 1
if number_of_tag_problems > 0:
console.log(f"Formatting issues: {number_of_tag_problems}")
EXIT_CODE = 1
sys.exit(EXIT_CODE)
sys.exit(0)