• 14.6k
• 5
• 29
• 204

• 1k
• 9
• 18

Context

General

The Python Packaging Authority (PyPA) provides a framework to easily build and install packages from a source project. It works nicely for Python code and text files. I choosed to use the modern hatchling which is a sister project of hatch as a build backend.

On the other end, the GNU project provides the gettext utilities to easily internationalize and localise an application.

So far so good, a gettext interface is natively available in the standard Python library, with a Python script able to compile a source .po file to a .mo one.

But there is no official tool to automatically compile those .po files at build time to have a neat and clean project.

Personal

In another project of mine, I need to compile some .po files. I had built a tool for that which could be called through the now deprecated python setup.py interface. I decided to make a hatchling plugin to solve the problem.

My project

Status

The plugin is fully written with a test coverage above 90% and is available at GitHub. But I would love it to be peer reviewed before declaring it production ready.

Description

(extracts from the project README file):

As it is only a hatchling hook plugin, this package has no direct interface. It uses the Application object provided by hatchling for its messages, so it is possible to increase its verbosity by just passing -v params to the hatch build command.

.po files

The source .po are expected to be in a folder under the root source directory. The default directory is messages but it can be changed through the configuration file. It can even be . if the files are directly in the root directory.

They are expected to be named LANG.po or domain-LANG.po. When no domain is present in the file name, the default domain is the last component of the source folder, or the package name if the source folder is . or messages.

LANG folders organization

Alternatively the source directory can contain LANG folders (e.g. fr_CA or es) that in turn contain domain.po files, possibly under a hierarchy of subfolders. The goal is to mimic a classical locale hierarchy: LANG/LC_MESSAGES/domain.po

.mo files

For every .po file found, a corresponding compiled file is generated as locale/LANG/LC_MESSAGES/domain.mo under the project root directory. The default locale name can be changed through the builder configuration.

Configuration

The hatch-msgfmt-s-ball plugin can be configured as any other plugin through the pyproject.toml file...

Source of the plugin file

"""
This module contains the implementation of a hatchling hook plugin.
This plugin allows to compile gettext .po files to .mo ones when building
a wheel and to install them under an appropriate (but local) directory.
"""
# required for 3.8 support
# TODO: can be removed as soon as 3.8 support will be dropped
from __future__ import annotations
import re
from pathlib import Path
from typing import Any, Generator
from .vendor.msgfmt import make
from hatchling.builders.hooks.plugin.interface import BuildHookInterface
class MsgFmtBuildHook(BuildHookInterface):
 """The implementation of the hook interface"""
 PLUGIN_NAME = "msgfmt" # required by the interface
 locale: Path # local folder for the gettext localedir folder
 src: Path # local folder for the source .po files
 # A compiled regex to split a file name into domain-lang
 full = re.compile('^(.*)-([a-z]{2}(?:_[A-Z]{2})?)$')
 def clean(self, _versions: list[str]) -> None:
 # Described in BuildHookInterface
 self.build_conf()
 self.app.display_debug('Cleaning everything in '
 + self.config['locale'], 2)
 force = self.config.get("force_clean")
 for name in sorted(self.locale.rglob('*'), reverse=True):
 if name.is_dir():
 try:
 name.rmdir()
 except OSError:
 self.app.display_warning(
 f'Folder {name.name} not removed (not empty?)')
 elif force or name.suffix == '.mo':
 try:
 name.unlink()
 except OSError:
 self.app.display_warning(f'File {name.name} not removed')
 def initialize(self, _version: str, build_data: dict[str, Any]) -> None:
 # Described in BuildHookInterface
 self.build_conf()
 self.app.display_debug(f'hatch-msgfmt-s-ball building {self.target_name}')
 if self.target_name != 'wheel':
 self.app.display_warning(
 f'{self.target_name}: unexpected target - call ignored')
 return
 if not self.src.is_dir():
 self.app.display_error(
 f'{self.config["messages"]} is not a directory: giving up')
 return
 for (path, lang, domain) in self.source_files():
 (self.locale / lang / 'LC_MESSAGES').mkdir(parents=True, exist_ok=True)
 mo = str(self.locale / lang / 'LC_MESSAGES' / (domain + '.mo'))
 make(str(path), mo)
 mox = 'locale/{lang}/LC_MESSAGES/{domain}.mo'.format(lang=lang,
 domain=domain)
 build_data['force_include'][mox] = mox
 self.app.display_debug('Compiling {src} to {locale}'.format(
 src=str(path), locale=mox
 ), 1)
 def build_conf(self) -> None:
 """
 Set default values for parameters not present in config files
 :return: None
 """
 if 'messages' not in self.config:
 self.config['messages'] = 'messages'
 if 'locale' not in self.config:
 self.config['locale'] = 'locale'
 self.locale = Path(self.root) / self.config['locale']
 self.src = Path(self.root) / self.config['messages']
 if 'domain' not in self.config:
 self.config['domain'] = (
 self.metadata.name
 if self.config['messages'] in ('.', 'messages')
 else self.src.name)
 def source_files(self) -> Generator[tuple[Path, str, str], None, None]:
 """
 Build a generator of tuples (file_path, lang, domain) of po files.
 :return: the generator
 """
 # a compiled regex to extract the domain if present and the lang code
 rx = re.compile(r'^(?:(.+)-)?([a-z]{2,3}(?:_[A-Z]+)?)$')
 for child in self.src.iterdir():
 if child.is_dir():
 # found a LANG folder
 for po in child.rglob(r'*.po'):
 yield po, child.name, po.stem
 elif child.suffix == '.po':
 m = rx.match(child.stem)
 if m:
 domain = (self.config['domain'] if m.group(1) is None
 else m.group(1))
 yield child, m.group(2), domain

Test module (using pytest):

"""
This pytest module does the heavy testing of the plugin module.
It ensures that every methode does the expected job when given the
appropriate parameters
"""
# required for 3.8 support
# TODO: can be removed as soon as 3.8 support will be dropped
from __future__ import annotations
import filecmp
import shutil
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Any, Union
from unittest.mock import Mock, PropertyMock, patch
from hatch_msgfmt import plugin
import pytest
from hatchling.bridge.app import Application
from hatchling.metadata.core import ProjectMetadata
from hatch_msgfmt.plugin import MsgFmtBuildHook
def build_hook(config: dict[str, Any]=None,
 target_name:str = 'wheel',
 directory: Union[str, Path] = '.',
 root: Union[str, Path] ='.'):
 """
 Builds a MsgFmtBuildHook with the correct parameters.
 The parameters BuilderConfig, ProjectMetadata and Application are
 always Mock values.
 :param config: a populated config or None
 :param target_name: the target name
 :param directory: the build directory (should be dist in the real world)
 :param root: the root directory (containing pyproject.toml in real)
 :return: a MsgFmtBuildHook
 """
 if config is None:
 config = {}
 from hatchling.builders.config import BuilderConfig
 return MsgFmtBuildHook(root, config, Mock(BuilderConfig),
 Mock(ProjectMetadata), directory, target_name,
 Mock(Application))
@pytest.fixture
def data_dir() -> Path:
 """
 pytest fixtures returning the pathlib.Path of the tests/data folder
 :return: the path of the tests/data folder
 """
 return Path(__file__).parent / 'data'
@pytest.fixture
def hook():
 """
 A pytest fixture returning a default MsgFmtBuildHook
 :return: a default MsgFmtBuildHook
 """
 return build_hook()
# noinspection PyUnresolvedReferences
def test_wrong_target():
 """
 Ensures that a warning is emitted is target is not wheel
 :return: None
 """
 hook = build_hook(target_name='sdist')
 hook.initialize('', {})
 hook.app.display_warning.assert_called()
 assert 'sdist' in hook.app.display_warning.call_args_list[0][0][0]
@pytest.fixture
def locale():
 """
 A yield pytest fixture providing a temporary locale folder.
 Using yield allows automatic removal of the temporary directory
 after the test.
 :return: a Path to a temporary locale folder
 """
 with TemporaryDirectory() as d:
 directory = Path(d, 'locale')
 directory.mkdir()
 yield directory
class TestClean:
 """
 Tests for the clean method of MsgFmtBuildHook
 """
 @pytest.fixture
 def hook(self, locale):
 """
 A specialized fixture providing a MsgFmtBuildHook having a
 locale directory
 :param locale: the Path to the locale folder
 :return: a MsgFmtBuildHook
 """
 return build_hook(root=str(locale.parent))
 def test_simple(self, locale, hook):
 """
 Ensures that a single mo file under the locale directory is removed
 :param locale: path to the locale directory
 :param hook: a configured MsgFmtBuildHook
 :return: None
 """
 fr = locale / 'fr'
 fr.mkdir()
 mo = fr / 'foo.mo'
 mo.write_bytes(b'abcd')
 hook.clean(['sdist', 'wheel'])
 assert len(list(locale.glob('*'))) == 0
 def test_other_than_mo(self, locale, hook):
 """
 Ensures that files without the .mo suffix are not removed.
 The folder containing them shall not be removed either
 :param locale: path to the locale folder
 :param hook: a configured MsgFmtBuildHook
 :return: None
 """
 fr = locale / 'fr'
 fr.mkdir()
 mo = fr / 'foo.mo'
 other = fr/'foo'
 mo.write_bytes(b'abcd')
 other.write_bytes(b'ef')
 assert len(list(locale.rglob('*'))) == 3
 hook.clean(['sdist', 'wheel'])
 # both fr/foo and fr shall remain...
 assert len(list(locale.rglob('*'))) == 2
 # noinspection PyUnresolvedReferences
 def test_unlink_error(self, locale, hook):
 """
 Ensures that an OS error when removing a .mo file issues a warning
 :param locale: path to the locale directory
 :param hook: a configured MsgFmtBuildHook
 :return: None
 """
 fr = locale / 'fr'
 fr.mkdir()
 mo = fr / 'foo.mo'
 mo.write_bytes(b'abcd')
 mo.chmod(0o444) # mark the foo.mo file as read only
 fr.chmod(0o555) # and its directory too...
 assert len(list(locale.rglob('*'))) == 2
 hook.clean(['sdist', 'wheel'])
 assert len(list(locale.rglob('*'))) == 2
 assert hook.app.display_warning.call_args_list[0][0][0].startswith('File')
 assert 'foo.mo' in hook.app.display_warning.call_args_list[0][0][0]
 def test_force(self, locale, hook):
 """
 Ensures that any file is removed with the force_clean option
 :param locale: path to the locale directory
 :param hook: a configured MsgFmtBuildHook
 :return: None
 """
 fr = locale / 'fr'
 fr.mkdir()
 mo = fr / 'foo.mo'
 other = fr/'foo'
 mo.write_bytes(b'abcd')
 other.write_bytes(b'ef')
 assert len(list(locale.rglob('*'))) == 3
 hook.config['force_clean'] = True
 hook.clean(['sdist', 'wheel'])
 assert len(list(locale.rglob('*'))) == 0
class TestDefaultDomain:
 """
 Test for detection and usage of a default gettext domain
 """
 # noinspection PyPropertyAccess
 def test_proj_name(self, hook):
 """
 Ensures that the default domain is the project name
 :param hook: a default MsgFmtBuildHook
 :return: None
 """
 type(hook.metadata).name = PropertyMock(return_value = 'proj_name')
 hook.build_conf()
 assert hook.config['domain'] == 'proj_name'
 def test_message(self):
 """
 Ensures that the default domain is the name of the source folder
 :return: None
 """
 hook = build_hook({'messages': 'dom'})
 hook.build_conf()
 assert hook.config['domain'] == 'dom'
 def test_domain(self):
 """
 Ensures that a specified domain name takes precedence
 :return: None
 """
 hook = build_hook({'messages': 'src', 'domain': 'dom'})
 hook.build_conf()
 assert hook.config['domain'] == 'dom'
@pytest.fixture
def messages(locale):
 """
 A pytest fixture that builds a messages folder as a sibling of the locale one
 :param locale: the result of the locale fixture
 :return: None
 """
 directory = locale.parent / 'messages'
 directory.mkdir()
 yield directory
class TestPoList:
 """
 Tests for the generation of the list of .po files
 """
 def test_flat_single_domain(self, messages):
 """
 Ensures that the list uses the default domain when the file name is LANG.po
 :param messages: a messages folder
 :return: None
 """
 po1 = messages / 'en.po'
 po1.write_text('#foo')
 po2 = po1.with_name('myapp-fr_CA.po')
 po2.write_text('#bar')
 hook = build_hook({'domain': 'myapp'}, root = str(messages.parent))
 hook.build_conf()
 lst = list(hook.source_files())
 assert len(lst) == 2
 assert (po1, 'en', 'myapp') in lst
 assert (po2, 'fr_CA', 'myapp') in lst
 def test_flat_many_domains(self, messages):
 """
 Ensures that a domain specified in the file name takes precedence
 :param messages: a messages folder
 :return: None
 """
 po1 = messages / 'en.po'
 po1.write_text('#foo')
 po2 = po1.with_name('foo-fr_CA.po')
 po2.write_text('#bar')
 hook = build_hook({'domain': 'myapp'}, root = str(messages.parent))
 hook.build_conf()
 lst = list(hook.source_files())
 assert len(lst) == 2
 assert (po1, 'en', 'myapp') in lst
 assert (po2, 'fr_CA', 'foo') in lst
 def test_lang_folders(self, messages):
 """
 Ensures that a language folder is used
 :param messages: a messages folder
 :return: None
 """
 fr = messages / 'fr_FR' / 'LC_MESSAGES'
 fr.mkdir(parents=True)
 de = messages /'de' / 'LC_MESSAGES'
 de.mkdir(parents=True)
 po1 = fr / 'myapp.po'
 po1.write_text('#foo')
 po2 = de / 'myapp.po'
 po2.write_text('#bar')
 hook = build_hook(root=messages.parent)
 hook.build_conf()
 lst = list(hook.source_files())
 assert len(lst) == 2
 assert (po1, 'fr_FR', 'myapp') in lst
 assert (po2, 'de', 'myapp') in lst
class TestFmt:
 """
 Tests for the generation of a .mo file by FmtMsg.py
 """
 def test_mocked(self, data_dir, messages):
 """
 Ensures (through a Mock) that the make function is correctly called
 :param data_dir: the tests/data folder containing a .po file
 :param messages: a messages folder
 :return: None
 """
 shutil.copy(data_dir / 'foo-fr.po', messages / 'foo-fr.po')
 hook = build_hook({'domain': 'foo'},root=messages.parent)
 build_data = {'force_include': {}}
 with patch('hatch_msgfmt.plugin.make'):
 hook.initialize('standard', build_data)
 # noinspection PyUnresolvedReferences
 plugin.make.assert_called_with(
 str(messages / 'foo-fr.po'),
 str(messages.parent / 'locale' / 'fr' / 'LC_MESSAGES' / 'foo.mo'))
 def test_flat(self, data_dir, messages, locale):
 """
 Ensures that the .mo files are generated in the correct folder
 Also ensures that the list of the generated .mo files is passed
 back for the calling hatchling builder.
 :param data_dir: the tests/data folder containing a .po file
 :param messages: a messages folder
 :param locale: the locale folder
 :return: None
 """
 shutil.copy(data_dir / 'foo-fr.po', messages / 'foo-fr.po')
 shutil.copy(data_dir / 'foo-fr.po', messages / 'bar-fr.po')
 shutil.copy(data_dir / 'foo-fr.po', messages / 'fr.po')
 hook = build_hook({'domain': 'fee'},root=messages.parent)
 build_data = {'force_include': {}}
 hook.initialize('standard', build_data)
 assert (locale / 'fr' / 'LC_MESSAGES').is_dir()
 assert (locale / 'fr' / 'LC_MESSAGES' / 'foo.mo').exists()
 assert (locale / 'fr' / 'LC_MESSAGES' / 'bar.mo').exists()
 assert (locale / 'fr' / 'LC_MESSAGES' / 'fee.mo').exists()
 assert filecmp.cmp(locale / 'fr' / 'LC_MESSAGES' / 'foo.mo',
 locale / 'fr' / 'LC_MESSAGES' / 'bar.mo', 0)
 assert filecmp.cmp(locale / 'fr' / 'LC_MESSAGES' / 'foo.mo',
 locale / 'fr' / 'LC_MESSAGES' / 'fee.mo', 0)
 assert {'locale/fr/LC_MESSAGES/foo.mo', 'locale/fr/LC_MESSAGES/bar.mo',
 'locale/fr/LC_MESSAGES/fee.mo'
 } == set(build_data['force_include'].values())
class TestGettext:
 """
 End-to-end test for the usage of the generated .mo file
 """
 def test_data(self, data_dir, messages, locale):
 """
 Ensures that the generated .mo file is correctly used by gettext
 :param data_dir: the tests/data folder containing a .po file
 :param messages: the source messages folder
 :param locale: the locale folder
 :return: None
 """
 import gettext
 shutil.copy(data_dir / 'foo-fr.po', messages / 'foo-fr.po')
 hook = build_hook({'domain': 'fee'},root=messages.parent)
 build_data = {'force_include': {}}
 hook.initialize('standard', build_data)
 assert gettext.find('foo', locale, ['fr_FR']) is not None
 trans = gettext.translation('foo', locale, ['fr_FR'])
 assert isinstance(trans, gettext.GNUTranslations)
 assert 'éè' == trans.gettext('foo')
 assert 'àç' == trans.ngettext('bar', 'baz', 1)
 assert 'ça' == trans.ngettext('bar', 'baz', 2)
 assert 'ça' == trans.ngettext('bar', 'baz', 0)

Other files not provided here: the project pyproject.toml file, an integration test file (test_launch.py) and the patched copy of the msgfmt.py file form the CPython source distribution.