Move lang files into the data directory

Author: pmac

bedrock/base/tests/test_accepted_locales.py

@@ -4,7 +4,7 @@
 from django.conf import settings
 from django.test import TestCase
 
-from bedrock.settings.base import get_dev_languages, path
+from bedrock.settings.base import get_dev_languages, data_path
 
 
 class AcceptedLocalesTest(TestCase):
@@ -15,8 +15,8 @@ class AcceptedLocalesTest(TestCase):
     DEV_LANGUAGES or PROD_LANGUAGES should be used.
 
     """
-    locale = path('locale')
-    locale_bkp = path('locale_bkp')
+    locale = data_path('locale')
+    locale_bkp = data_path('locale_bkp')
 
     @classmethod
     def setup_class(cls):

bedrock/settings/base.py

@@ -19,17 +19,18 @@
 
 
 # ROOT path of the project. A pathlib.Path object.
+DATA_PATH = config('DATA_PATH', default='data')
 ROOT_PATH = Path(__file__).resolve().parents[2]
-GIT_REPOS_PATH = ROOT_PATH / 'git-repos'
+DATA_PATH = ROOT_PATH / DATA_PATH
 ROOT = str(ROOT_PATH)
 
 
 def path(*args):
     return abspath(str(ROOT_PATH.joinpath(*args)))
 
 
-def git_repo_path(*args):
-    return abspath(str(GIT_REPOS_PATH.joinpath(*args)))
+def data_path(*args):
+    return abspath(str(DATA_PATH.joinpath(*args)))
 
 
 # Is this a dev instance?
@@ -41,7 +42,7 @@ def git_repo_path(*args):
 DATABASES = {
     'default': {
         'ENGINE': 'django_prometheus.db.backends.sqlite3',
-        'NAME': path('bedrock.db'),
+        'NAME': data_path('bedrock.db'),
     },
 }
 
@@ -112,7 +113,7 @@ def git_repo_path(*args):
                               default='product_details.storage.PDDatabaseStorage')
 # path into which to clone the p-d json repo
 PROD_DETAILS_JSON_REPO_PATH = config('PROD_DETAILS_JSON_REPO_PATH',
-                                     default=git_repo_path('product_details_json'))
+                                     default=data_path('product_details_json'))
 PROD_DETAILS_JSON_REPO_URI = config('PROD_DETAILS_JSON_REPO_URI',
                                     default='https://github.com/mozilla-releng/product-details.git')
 PROD_DETAILS_JSON_REPO_BRANCH = config('PROD_DETAILS_JSON_REPO_BRANCH', default='production')
@@ -134,7 +135,7 @@ def git_repo_path(*args):
                   'sr', 'sv-SE', 'ta', 'te', 'th', 'tl', 'tr', 'trs', 'uk', 'ur',
                   'uz', 'vi', 'xh', 'zh-CN', 'zh-TW', 'zu')
 
-LOCALES_PATH = ROOT_PATH / 'locale'
+LOCALES_PATH = DATA_PATH / 'locale'
 default_locales_repo = 'www.mozilla.org' if DEV else 'bedrock-l10n'
 default_locales_repo = 'https://github.com/mozilla-l10n/{}'.format(default_locales_repo)
 LOCALES_REPO = config('LOCALES_REPO', default=default_locales_repo)
@@ -260,13 +261,13 @@ def lazy_langs():
 FLUENT_DEFAULT_PERCENT_REQUIRED = config('FLUENT_DEFAULT_PERCENT_REQUIRED', default='80', parser=int)
 FLUENT_REPO = config('FLUENT_REPO', default='mozmeao/www-l10n')
 FLUENT_REPO_URL = f'https://github.com/{FLUENT_REPO}'
-FLUENT_REPO_PATH = GIT_REPOS_PATH / 'www-l10n'
+FLUENT_REPO_PATH = DATA_PATH / 'www-l10n'
 # will be something like "<github username>:<github token>"
 FLUENT_REPO_AUTH = config('FLUENT_REPO_AUTH', default='')
 FLUENT_LOCAL_PATH = ROOT_PATH / 'l10n'
 FLUENT_L10N_TEAM_REPO = config('FLUENT_L10N_TEAM_REPO', default='mozilla-l10n/www-l10n')
 FLUENT_L10N_TEAM_REPO_URL = f'https://github.com/{FLUENT_L10N_TEAM_REPO}'
-FLUENT_L10N_TEAM_REPO_PATH = GIT_REPOS_PATH / 'l10n-team'
+FLUENT_L10N_TEAM_REPO_PATH = DATA_PATH / 'l10n-team'
 # 10 seconds during dev and 10 min in prod
 FLUENT_CACHE_TIMEOUT = config('FLUENT_CACHE_TIMEOUT', default='10' if DEBUG else '600', parser=int)
 # order matters. first sting found wins.
@@ -703,7 +704,7 @@ def get_app_name(hostname):
 # Google Analytics
 GA_ACCOUNT_CODE = ''
 
-EXTERNAL_FILES_PATH = config('EXTERNAL_FILES_PATH', default=git_repo_path('community_data'))
+EXTERNAL_FILES_PATH = config('EXTERNAL_FILES_PATH', default=data_path('community_data'))
 EXTERNAL_FILES_BRANCH = config('EXTERNAL_FILES_BRANCH', default='master')
 EXTERNAL_FILES_REPO = config('EXTERNAL_FILES_REPO', default='https://github.com/mozilla/community-data.git')
 EXTERNAL_FILES = {
@@ -1352,16 +1353,16 @@ def get_app_name(hostname):
 else:
     content_cards_default_branch = 'prod-processed'
 
-CONTENT_CARDS_PATH = config('CONTENT_CARDS_PATH', default=git_repo_path('content_cards'))
+CONTENT_CARDS_PATH = config('CONTENT_CARDS_PATH', default=data_path('content_cards'))
 CONTENT_CARDS_REPO = config('CONTENT_CARDS_REPO', default='https://github.com/mozmeao/www-admin.git')
 CONTENT_CARDS_BRANCH = config('CONTENT_CARDS_BRANCH', default=content_cards_default_branch)
 CONTENT_CARDS_URL = config('CONTENT_CARDS_URL', default=STATIC_URL)
 
-RELEASE_NOTES_PATH = config('RELEASE_NOTES_PATH', default=git_repo_path('release_notes'))
+RELEASE_NOTES_PATH = config('RELEASE_NOTES_PATH', default=data_path('release_notes'))
 RELEASE_NOTES_REPO = config('RELEASE_NOTES_REPO', default='https://github.com/mozilla/release-notes.git')
 RELEASE_NOTES_BRANCH = config('RELEASE_NOTES_BRANCH', default='master')
 
-WWW_CONFIG_PATH = config('WWW_CONFIG_PATH', default=git_repo_path('www_config'))
+WWW_CONFIG_PATH = config('WWW_CONFIG_PATH', default=data_path('www_config'))
 WWW_CONFIG_REPO = config('WWW_CONFIG_REPO', default='https://github.com/mozmeao/www-config.git')
 WWW_CONFIG_BRANCH = config('WWW_CONFIG_BRANCH', default='master')
 
@@ -1372,7 +1373,7 @@ def get_app_name(hostname):
 LEGAL_DOCS_CACHE_TIMEOUT = config('LEGAL_DOCS_CACHE_TIMEOUT', default='60' if DEV else '600', parser=int)
 
 MOFO_SECURITY_ADVISORIES_PATH = config('MOFO_SECURITY_ADVISORIES_PATH',
-                                       default=git_repo_path('mofo_security_advisories'))
+                                       default=data_path('mofo_security_advisories'))
 MOFO_SECURITY_ADVISORIES_REPO = config('MOFO_SECURITY_ADVISORIES_REPO',
                                        default='https://github.com/mozilla/'
                                                'foundation-security-advisories.git')

bin/db_s3_utils.py

@@ -4,8 +4,9 @@
 from subprocess import check_output, CalledProcessError
 
 
-JSON_DATA_FILE = getenv('AWS_DB_JSON_DATA_FILE', 'bedrock_db_info.json')
-DB_FILE = 'bedrock.db'
+DATA_PATH = getenv('DATA_PATH', 'data')
+JSON_DATA_FILE = getenv('AWS_DB_JSON_DATA_FILE', f'{DATA_PATH}/bedrock_db_info.json')
+DB_FILE = f'{DATA_PATH}/bedrock.db'
 CACHE = {}
 BLOCKSIZE = 65536
 

bin/run-db-download.py

@@ -12,6 +12,7 @@
     set_db_data,
     JSON_DATA_FILE,
     DB_FILE,
+    DATA_PATH,
 )
 
 
@@ -29,7 +30,7 @@ def get_file_url(filename):
 
 def download_db_info():
     try:
-        resp = requests.get(get_file_url(JSON_DATA_FILE))
+        resp = requests.get(get_file_url(os.path.basename(JSON_DATA_FILE)))
         resp.raise_for_status()
     except requests.RequestException:
         return None
@@ -42,7 +43,7 @@ def download_db_info():
 
 
 def download_db_file(filename):
-    resp = requests.get(get_file_url(filename), stream=True)
+    resp = requests.get(get_file_url(os.path.basename(filename)), stream=True)
     with open(filename, 'wb') as fp:
         for chunk in resp.iter_content(chunk_size=128):
             fp.write(chunk)
@@ -76,6 +77,7 @@ def main(args):
                 return 0
 
     new_db_file = db_info['file_name']
+    new_db_file = f'{DATA_PATH}/{new_db_file}'
     download_db_file(new_db_file)
     checksum = get_db_checksum(new_db_file)
     if checksum == db_info['checksum']:

bin/run-sync-all.sh

@@ -6,5 +6,8 @@ if [ ! -e ./manage.py ]; then
     cd $script_parent_dir
 fi
 
+# ensure the data dir exists
+mkdir -p data
+
 # use honcho to inject the proper env vars
 honcho run --env docker/envfiles/prod.env ./bin/sync-all.sh

data/.gitignore

@@ -0,0 +1,2 @@
+**/*
+!.gitignore

docker-compose.yml

@@ -43,7 +43,6 @@ services:
       - ./wsgi/:/app/wsgi:delegated
       - ./locale/:/app/locale:delegated
       - ./l10n/:/app/l10n:delegated
-      - ./git-repos/:/app/git-repos:delegated
 
   # run the tests against local changes
   test:
@@ -61,7 +60,6 @@ services:
       - ./tests/:/app/tests:delegated
       - ./wsgi/:/app/wsgi:delegated
       - ./l10n/:/app/l10n:delegated
-      - ./git-repos/:/app/git-repos:delegated
 
   # run tests with no volumes
   test-image:
@@ -93,7 +91,6 @@ services:
       - ./wsgi/:/app/wsgi:delegated
       - ./locale/:/app/locale:delegated
       - ./l10n/:/app/l10n:delegated
-      - ./git-repos/:/app/git-repos:delegated
 
   builder:
     build:

docs/fluent-conversion.rst

@@ -163,7 +163,7 @@ recipe path.
 Before you run this, make sure to update the local clones of your l10n repositories.
 
 This command will use the Fluent file you edited as template, read the legacy translations
-from ``locale`` and write the generated Fluent files for each locale into the ``git-repos/www-l10n/`` directory.
+from ``locale`` and write the generated Fluent files for each locale into the ``data/www-l10n/`` directory.
 
 Append each locale you wish to convert in a space-separated list. You can find the full
 list of supported locales in the `Pontoon configuration file`_ and below in a space-separated

docs/l10n.rst

@@ -20,7 +20,7 @@ If you don't already have a ``locale`` directory, this command will clone the
 git repo containing the .lang translation files (either the dev or prod files
 depending on your ``DEV`` setting). If the folder is already present, it will
 update the repository to the latest version. It do the same thing for the
-repository for the .ftl translation files in ``git-repos/www-l10n`` directory.
+repository for the .ftl translation files in ``data/www-l10n`` directory.
 
 Fluent
 ======

lib/l10n_utils/dotlang.py

@@ -114,11 +114,9 @@ def translate(text, files):
 
     for file_ in files:
         key = "dotlang-%s-%s" % (lang, file_)
-        rel_path = os.path.join('locale', lang, '%s.lang' % file_)
-
+        path = str(settings.LOCALES_PATH / lang / f'{file_}.lang')
         trans = cache.get(key)
         if trans is None:
-            path = os.path.join(settings.ROOT, rel_path)
             trans = parse(path)
             cache.set(key, trans)
 
@@ -130,7 +128,7 @@ def translate(text, files):
                                'replaced text (aka %s)')
                 message = '%s\n\n%s\n%s' % (explanation, text,
                                             trans[tweaked_text])
-                mail_error(rel_path, message)
+                mail_error(path, message)
                 return Markup(text)
             return Markup(trans[tweaked_text])
     return Markup(text)
@@ -225,12 +223,11 @@ def lang_file_tag_set(path, lang=None):
         return ALL_THE_THINGS
 
     lang = lang or translation.get_language(True)
-    rel_path = os.path.join('locale', lang, '%s.lang' % path)
-    cache_key = 'tag:%s' % rel_path
+    fpath = settings.LOCALES_PATH / lang / f'{path}.lang'
+    cache_key = f'tag:locale/{lang}/{path}.lang'
     tag_set = cache.get(cache_key)
     if tag_set is None:
         tag_set = set()
-        fpath = os.path.join(settings.ROOT, rel_path)
         try:
             with codecs.open(fpath, 'r', 'utf-8', errors='replace') as lines:
                 for line in lines:

lib/l10n_utils/gettext.py

@@ -76,18 +76,18 @@ def extract_content(s):
 
 
 def po_msgs(domain):
-    return parse_po(join(settings.ROOT, 'locale', 'templates', 'LC_MESSAGES',
-                         '{}.pot'.format(domain)))
+    path = settings.LOCALES_PATH / 'templates' / 'LC_MESSAGES' / f'{domain}.pot'
+    return parse_po(str(path))
 
 
 def translated_strings(file_):
-    path = join(settings.ROOT, 'locale', 'templates', file_)
+    path = str(settings.LOCALES_PATH / 'templates', file_)
     trans = list(parse_lang(path, skip_untranslated=False).keys())
     return trans
 
 
 def lang_file(name, lang):
-    return join(settings.ROOT, 'locale', lang, name)
+    return str(settings.LOCALES_PATH / lang / name)
 
 
 def is_template(path):

lib/l10n_utils/management/commands/l10n_check.py

@@ -20,7 +20,7 @@
 
 
 def l10n_file(*args):
-    return path.join(settings.ROOT, 'locale', *args)
+    return str(settings.LOCALES_PATH.joinpath(*args))
 
 
 def list_templates():

lib/l10n_utils/management/commands/l10n_extract.py

@@ -150,8 +150,7 @@ def handle(self, *args, **options):
         filenames = options['filenames']
         if filenames:
             # mimics puente.management.commands.extract for a list of files
-            outputdir = os.path.join(settings.ROOT, 'locale', 'templates',
-                                     'LC_MESSAGES')
+            outputdir = str(settings.LOCALES_PATH / 'templates' / 'LC_MESSAGES')
             if not os.path.isdir(outputdir):
                 os.makedirs(outputdir)
 

lib/l10n_utils/management/commands/l10n_merge.py

@@ -20,7 +20,7 @@ def add_arguments(self, parser):
     def handle(self, *args, **options):
         langs = options['langs']
         if not langs:
-            langs = os.listdir(os.path.join(settings.ROOT, 'locale'))
+            langs = os.listdir(str(settings.LOCALES_PATH))
             langs = [x for x in langs if x != 'templates']
             langs = [x for x in langs if x[0] != '.']
 

lib/l10n_utils/tests/test_commands.py

@@ -6,6 +6,7 @@
 import codecs
 from os import path
 from io import StringIO, IOBase
+from pathlib import Path
 from textwrap import dedent
 
 from django.conf import settings
@@ -23,9 +24,10 @@
 from lib.l10n_utils.management.commands.l10n_extract import extract_from_files
 from lib.l10n_utils.tests import capture_stdio
 
-
-ROOT = path.join(path.dirname(path.abspath(__file__)), 'test_files')
-TEMPLATE_DIRS = (path.join(ROOT, 'templates'),)
+ROOT_PATH = Path(__file__).with_name('test_files')
+ROOT = str(ROOT_PATH)
+LOCALES_PATH = ROOT_PATH / 'locale'
+TEMPLATE_DIRS = (str(ROOT_PATH / 'templates'),)
 
 METHODS = [
     ('templates/**.html',
@@ -38,7 +40,7 @@
 TRUE_MOCK.return_value = True
 
 
-@override_settings(ROOT=ROOT)
+@override_settings(ROOT=ROOT, LOCALES_PATH=LOCALES_PATH)
 class TestL10nExtract(TestCase):
     def test_extract_from_files(self):
         """
@@ -116,7 +118,7 @@ def test_extract_from_files_callback_works(self):
         callback.assert_called_once_with(testfile[0], METHODS[0][1], ANY)
 
 
-@override_settings(ROOT=ROOT)
+@override_settings(LOCALES_PATH=LOCALES_PATH)
 class TestL10nCheck(TestCase):
     def _get_block(self, blocks, name):
         """Out of all blocks, grab the one with the specified name."""
@@ -328,7 +330,7 @@ def test_copy_template(self):
         self.assertEqual(open_buffer.getvalue(), good_value)
 
 
-@override_settings(ROOT=ROOT)
+@override_settings(LOCALES_PATH=LOCALES_PATH)
 class Testl10nMerge(TestCase):
     @patch('lib.l10n_utils.gettext.settings.ROOT', ROOT)
     @patch('lib.l10n_utils.gettext._append_to_lang_file')
@@ -338,7 +340,7 @@ def test_merge_lang_files(self, write_mock):
         Bug 861168.
         """
         merge_lang_files(['de'])
-        dest_file = path.join(ROOT, 'locale', 'de', 'firefox', 'fx.lang')
+        dest_file = str(LOCALES_PATH / 'de' / 'firefox' / 'fx.lang')
         write_mock.assert_called_once_with(dest_file,
                                            [[None, u'Find out if your device is '
                                                    u'supported  \xbb']])