843 lines
34 KiB
Python
843 lines
34 KiB
Python
|
"""
|
||
|
|
babel.messages.extract
|
||
|
|
~~~~~~~~~~~~~~~~~~~~~~
|
||
|
|
|
||
|
|
Basic infrastructure for extracting localizable messages from source files.
|
||
|
|
|
||
|
|
This module defines an extensible system for collecting localizable message
|
||
|
|
strings from a variety of sources. A native extractor for Python source
|
||
|
|
files is builtin, extractors for other sources can be added using very
|
||
|
|
simple plugins.
|
||
|
|
|
||
|
|
The main entry points into the extraction functionality are the functions
|
||
|
|
`extract_from_dir` and `extract_from_file`.
|
||
|
|
|
||
|
|
:copyright: (c) 2013-2023 by the Babel Team.
|
||
|
|
:license: BSD, see LICENSE for more details.
|
||
|
|
"""
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import ast
|
||
|
|
import io
|
||
|
|
import os
|
||
|
|
import sys
|
||
|
|
import tokenize
|
||
|
|
from collections.abc import (
|
||
|
|
Callable,
|
||
|
|
Collection,
|
||
|
|
Generator,
|
||
|
|
Iterable,
|
||
|
|
Mapping,
|
||
|
|
MutableSequence,
|
||
|
|
)
|
||
|
|
from os.path import relpath
|
||
|
|
from textwrap import dedent
|
||
|
|
from tokenize import COMMENT, NAME, OP, STRING, generate_tokens
|
||
|
|
from typing import TYPE_CHECKING, Any
|
||
|
|
|
||
|
|
from babel.util import parse_encoding, parse_future_flags, pathmatch
|
||
|
|
|
||
|
|
if TYPE_CHECKING:
|
||
|
|
from typing import IO, Protocol
|
||
|
|
|
||
|
|
from _typeshed import SupportsItems, SupportsRead, SupportsReadline
|
||
|
|
from typing_extensions import Final, TypeAlias, TypedDict
|
||
|
|
|
||
|
|
class _PyOptions(TypedDict, total=False):
|
||
|
|
encoding: str
|
||
|
|
|
||
|
|
class _JSOptions(TypedDict, total=False):
|
||
|
|
encoding: str
|
||
|
|
jsx: bool
|
||
|
|
template_string: bool
|
||
|
|
parse_template_string: bool
|
||
|
|
|
||
|
|
class _FileObj(SupportsRead[bytes], SupportsReadline[bytes], Protocol):
|
||
|
|
def seek(self, __offset: int, __whence: int = ...) -> int: ...
|
||
|
|
def tell(self) -> int: ...
|
||
|
|
|
||
|
|
_SimpleKeyword: TypeAlias = tuple[int | tuple[int, int] | tuple[int, str], ...] | None
|
||
|
|
_Keyword: TypeAlias = dict[int | None, _SimpleKeyword] | _SimpleKeyword
|
||
|
|
|
||
|
|
# 5-tuple of (filename, lineno, messages, comments, context)
|
||
|
|
_FileExtractionResult: TypeAlias = tuple[str, int, str | tuple[str, ...], list[str], str | None]
|
||
|
|
|
||
|
|
# 4-tuple of (lineno, message, comments, context)
|
||
|
|
_ExtractionResult: TypeAlias = tuple[int, str | tuple[str, ...], list[str], str | None]
|
||
|
|
|
||
|
|
# Required arguments: fileobj, keywords, comment_tags, options
|
||
|
|
# Return value: Iterable of (lineno, message, comments, context)
|
||
|
|
_CallableExtractionMethod: TypeAlias = Callable[
|
||
|
|
[_FileObj | IO[bytes], Mapping[str, _Keyword], Collection[str], Mapping[str, Any]],
|
||
|
|
Iterable[_ExtractionResult],
|
||
|
|
]
|
||
|
|
|
||
|
|
_ExtractionMethod: TypeAlias = _CallableExtractionMethod | str
|
||
|
|
|
||
|
|
GROUP_NAME: Final[str] = 'babel.extractors'
|
||
|
|
|
||
|
|
DEFAULT_KEYWORDS: dict[str, _Keyword] = {
|
||
|
|
'_': None,
|
||
|
|
'gettext': None,
|
||
|
|
'ngettext': (1, 2),
|
||
|
|
'ugettext': None,
|
||
|
|
'ungettext': (1, 2),
|
||
|
|
'dgettext': (2,),
|
||
|
|
'dngettext': (2, 3),
|
||
|
|
'N_': None,
|
||
|
|
'pgettext': ((1, 'c'), 2),
|
||
|
|
'npgettext': ((1, 'c'), 2, 3),
|
||
|
|
}
|
||
|
|
|
||
|
|
DEFAULT_MAPPING: list[tuple[str, str]] = [('**.py', 'python')]
|
||
|
|
|
||
|
|
# New tokens in Python 3.12, or None on older versions
|
||
|
|
FSTRING_START = getattr(tokenize, "FSTRING_START", None)
|
||
|
|
FSTRING_MIDDLE = getattr(tokenize, "FSTRING_MIDDLE", None)
|
||
|
|
FSTRING_END = getattr(tokenize, "FSTRING_END", None)
|
||
|
|
|
||
|
|
|
||
|
|
def _strip_comment_tags(comments: MutableSequence[str], tags: Iterable[str]):
|
||
|
|
"""Helper function for `extract` that strips comment tags from strings
|
||
|
|
in a list of comment lines. This functions operates in-place.
|
||
|
|
"""
|
||
|
|
def _strip(line: str):
|
||
|
|
for tag in tags:
|
||
|
|
if line.startswith(tag):
|
||
|
|
return line[len(tag):].strip()
|
||
|
|
return line
|
||
|
|
comments[:] = map(_strip, comments)
|
||
|
|
|
||
|
|
|
||
|
|
def default_directory_filter(dirpath: str | os.PathLike[str]) -> bool:
|
||
|
|
subdir = os.path.basename(dirpath)
|
||
|
|
# Legacy default behavior: ignore dot and underscore directories
|
||
|
|
return not (subdir.startswith('.') or subdir.startswith('_'))
|
||
|
|
|
||
|
|
|
||
|
|
def extract_from_dir(
|
||
|
|
dirname: str | os.PathLike[str] | None = None,
|
||
|
|
method_map: Iterable[tuple[str, str]] = DEFAULT_MAPPING,
|
||
|
|
options_map: SupportsItems[str, dict[str, Any]] | None = None,
|
||
|
|
keywords: Mapping[str, _Keyword] = DEFAULT_KEYWORDS,
|
||
|
|
comment_tags: Collection[str] = (),
|
||
|
|
callback: Callable[[str, str, dict[str, Any]], object] | None = None,
|
||
|
|
strip_comment_tags: bool = False,
|
||
|
|
directory_filter: Callable[[str], bool] | None = None,
|
||
|
|
) -> Generator[_FileExtractionResult, None, None]:
|
||
|
|
"""Extract messages from any source files found in the given directory.
|
||
|
|
|
||
|
|
This function generates tuples of the form ``(filename, lineno, message,
|
||
|
|
comments, context)``.
|
||
|
|
|
||
|
|
Which extraction method is used per file is determined by the `method_map`
|
||
|
|
parameter, which maps extended glob patterns to extraction method names.
|
||
|
|
For example, the following is the default mapping:
|
||
|
|
|
||
|
|
>>> method_map = [
|
||
|
|
... ('**.py', 'python')
|
||
|
|
... ]
|
||
|
|
|
||
|
|
This basically says that files with the filename extension ".py" at any
|
||
|
|
level inside the directory should be processed by the "python" extraction
|
||
|
|
method. Files that don't match any of the mapping patterns are ignored. See
|
||
|
|
the documentation of the `pathmatch` function for details on the pattern
|
||
|
|
syntax.
|
||
|
|
|
||
|
|
The following extended mapping would also use the "genshi" extraction
|
||
|
|
method on any file in "templates" subdirectory:
|
||
|
|
|
||
|
|
>>> method_map = [
|
||
|
|
... ('**/templates/**.*', 'genshi'),
|
||
|
|
... ('**.py', 'python')
|
||
|
|
... ]
|
||
|
|
|
||
|
|
The dictionary provided by the optional `options_map` parameter augments
|
||
|
|
these mappings. It uses extended glob patterns as keys, and the values are
|
||
|
|
dictionaries mapping options names to option values (both strings).
|
||
|
|
|
||
|
|
The glob patterns of the `options_map` do not necessarily need to be the
|
||
|
|
same as those used in the method mapping. For example, while all files in
|
||
|
|
the ``templates`` folders in an application may be Genshi applications, the
|
||
|
|
options for those files may differ based on extension:
|
||
|
|
|
||
|
|
>>> options_map = {
|
||
|
|
... '**/templates/**.txt': {
|
||
|
|
... 'template_class': 'genshi.template:TextTemplate',
|
||
|
|
... 'encoding': 'latin-1'
|
||
|
|
... },
|
||
|
|
... '**/templates/**.html': {
|
||
|
|
... 'include_attrs': ''
|
||
|
|
... }
|
||
|
|
... }
|
||
|
|
|
||
|
|
:param dirname: the path to the directory to extract messages from. If
|
||
|
|
not given the current working directory is used.
|
||
|
|
:param method_map: a list of ``(pattern, method)`` tuples that maps of
|
||
|
|
extraction method names to extended glob patterns
|
||
|
|
:param options_map: a dictionary of additional options (optional)
|
||
|
|
:param keywords: a dictionary mapping keywords (i.e. names of functions
|
||
|
|
that should be recognized as translation functions) to
|
||
|
|
tuples that specify which of their arguments contain
|
||
|
|
localizable strings
|
||
|
|
:param comment_tags: a list of tags of translator comments to search for
|
||
|
|
and include in the results
|
||
|
|
:param callback: a function that is called for every file that message are
|
||
|
|
extracted from, just before the extraction itself is
|
||
|
|
performed; the function is passed the filename, the name
|
||
|
|
of the extraction method and and the options dictionary as
|
||
|
|
positional arguments, in that order
|
||
|
|
:param strip_comment_tags: a flag that if set to `True` causes all comment
|
||
|
|
tags to be removed from the collected comments.
|
||
|
|
:param directory_filter: a callback to determine whether a directory should
|
||
|
|
be recursed into. Receives the full directory path;
|
||
|
|
should return True if the directory is valid.
|
||
|
|
:see: `pathmatch`
|
||
|
|
"""
|
||
|
|
if dirname is None:
|
||
|
|
dirname = os.getcwd()
|
||
|
|
if options_map is None:
|
||
|
|
options_map = {}
|
||
|
|
if directory_filter is None:
|
||
|
|
directory_filter = default_directory_filter
|
||
|
|
|
||
|
|
absname = os.path.abspath(dirname)
|
||
|
|
for root, dirnames, filenames in os.walk(absname):
|
||
|
|
dirnames[:] = [
|
||
|
|
subdir for subdir in dirnames
|
||
|
|
if directory_filter(os.path.join(root, subdir))
|
||
|
|
]
|
||
|
|
dirnames.sort()
|
||
|
|
filenames.sort()
|
||
|
|
for filename in filenames:
|
||
|
|
filepath = os.path.join(root, filename).replace(os.sep, '/')
|
||
|
|
|
||
|
|
yield from check_and_call_extract_file(
|
||
|
|
filepath,
|
||
|
|
method_map,
|
||
|
|
options_map,
|
||
|
|
callback,
|
||
|
|
keywords,
|
||
|
|
comment_tags,
|
||
|
|
strip_comment_tags,
|
||
|
|
dirpath=absname,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def check_and_call_extract_file(
|
||
|
|
filepath: str | os.PathLike[str],
|
||
|
|
method_map: Iterable[tuple[str, str]],
|
||
|
|
options_map: SupportsItems[str, dict[str, Any]],
|
||
|
|
callback: Callable[[str, str, dict[str, Any]], object] | None,
|
||
|
|
keywords: Mapping[str, _Keyword],
|
||
|
|
comment_tags: Collection[str],
|
||
|
|
strip_comment_tags: bool,
|
||
|
|
dirpath: str | os.PathLike[str] | None = None,
|
||
|
|
) -> Generator[_FileExtractionResult, None, None]:
|
||
|
|
"""Checks if the given file matches an extraction method mapping, and if so, calls extract_from_file.
|
||
|
|
|
||
|
|
Note that the extraction method mappings are based relative to dirpath.
|
||
|
|
So, given an absolute path to a file `filepath`, we want to check using
|
||
|
|
just the relative path from `dirpath` to `filepath`.
|
||
|
|
|
||
|
|
Yields 5-tuples (filename, lineno, messages, comments, context).
|
||
|
|
|
||
|
|
:param filepath: An absolute path to a file that exists.
|
||
|
|
:param method_map: a list of ``(pattern, method)`` tuples that maps of
|
||
|
|
extraction method names to extended glob patterns
|
||
|
|
:param options_map: a dictionary of additional options (optional)
|
||
|
|
:param callback: a function that is called for every file that message are
|
||
|
|
extracted from, just before the extraction itself is
|
||
|
|
performed; the function is passed the filename, the name
|
||
|
|
of the extraction method and and the options dictionary as
|
||
|
|
positional arguments, in that order
|
||
|
|
:param keywords: a dictionary mapping keywords (i.e. names of functions
|
||
|
|
that should be recognized as translation functions) to
|
||
|
|
tuples that specify which of their arguments contain
|
||
|
|
localizable strings
|
||
|
|
:param comment_tags: a list of tags of translator comments to search for
|
||
|
|
and include in the results
|
||
|
|
:param strip_comment_tags: a flag that if set to `True` causes all comment
|
||
|
|
tags to be removed from the collected comments.
|
||
|
|
:param dirpath: the path to the directory to extract messages from.
|
||
|
|
:return: iterable of 5-tuples (filename, lineno, messages, comments, context)
|
||
|
|
:rtype: Iterable[tuple[str, int, str|tuple[str], list[str], str|None]
|
||
|
|
"""
|
||
|
|
# filename is the relative path from dirpath to the actual file
|
||
|
|
filename = relpath(filepath, dirpath)
|
||
|
|
|
||
|
|
for pattern, method in method_map:
|
||
|
|
if not pathmatch(pattern, filename):
|
||
|
|
continue
|
||
|
|
|
||
|
|
options = {}
|
||
|
|
for opattern, odict in options_map.items():
|
||
|
|
if pathmatch(opattern, filename):
|
||
|
|
options = odict
|
||
|
|
if callback:
|
||
|
|
callback(filename, method, options)
|
||
|
|
for message_tuple in extract_from_file(
|
||
|
|
method, filepath,
|
||
|
|
keywords=keywords,
|
||
|
|
comment_tags=comment_tags,
|
||
|
|
options=options,
|
||
|
|
strip_comment_tags=strip_comment_tags,
|
||
|
|
):
|
||
|
|
yield (filename, *message_tuple)
|
||
|
|
|
||
|
|
break
|
||
|
|
|
||
|
|
|
||
|
|
def extract_from_file(
|
||
|
|
method: _ExtractionMethod,
|
||
|
|
filename: str | os.PathLike[str],
|
||
|
|
keywords: Mapping[str, _Keyword] = DEFAULT_KEYWORDS,
|
||
|
|
comment_tags: Collection[str] = (),
|
||
|
|
options: Mapping[str, Any] | None = None,
|
||
|
|
strip_comment_tags: bool = False,
|
||
|
|
) -> list[_ExtractionResult]:
|
||
|
|
"""Extract messages from a specific file.
|
||
|
|
|
||
|
|
This function returns a list of tuples of the form ``(lineno, message, comments, context)``.
|
||
|
|
|
||
|
|
:param filename: the path to the file to extract messages from
|
||
|
|
:param method: a string specifying the extraction method (.e.g. "python")
|
||
|
|
:param keywords: a dictionary mapping keywords (i.e. names of functions
|
||
|
|
that should be recognized as translation functions) to
|
||
|
|
tuples that specify which of their arguments contain
|
||
|
|
localizable strings
|
||
|
|
:param comment_tags: a list of translator tags to search for and include
|
||
|
|
in the results
|
||
|
|
:param strip_comment_tags: a flag that if set to `True` causes all comment
|
||
|
|
tags to be removed from the collected comments.
|
||
|
|
:param options: a dictionary of additional options (optional)
|
||
|
|
:returns: list of tuples of the form ``(lineno, message, comments, context)``
|
||
|
|
:rtype: list[tuple[int, str|tuple[str], list[str], str|None]
|
||
|
|
"""
|
||
|
|
if method == 'ignore':
|
||
|
|
return []
|
||
|
|
|
||
|
|
with open(filename, 'rb') as fileobj:
|
||
|
|
return list(extract(method, fileobj, keywords, comment_tags,
|
||
|
|
options, strip_comment_tags))
|
||
|
|
|
||
|
|
|
||
|
|
def _match_messages_against_spec(lineno: int, messages: list[str|None], comments: list[str],
|
||
|
|
fileobj: _FileObj, spec: tuple[int|tuple[int, str], ...]):
|
||
|
|
translatable = []
|
||
|
|
context = None
|
||
|
|
|
||
|
|
# last_index is 1 based like the keyword spec
|
||
|
|
last_index = len(messages)
|
||
|
|
for index in spec:
|
||
|
|
if isinstance(index, tuple): # (n, 'c')
|
||
|
|
context = messages[index[0] - 1]
|
||
|
|
continue
|
||
|
|
if last_index < index:
|
||
|
|
# Not enough arguments
|
||
|
|
return
|
||
|
|
message = messages[index - 1]
|
||
|
|
if message is None:
|
||
|
|
return
|
||
|
|
translatable.append(message)
|
||
|
|
|
||
|
|
# keyword spec indexes are 1 based, therefore '-1'
|
||
|
|
if isinstance(spec[0], tuple):
|
||
|
|
# context-aware *gettext method
|
||
|
|
first_msg_index = spec[1] - 1
|
||
|
|
else:
|
||
|
|
first_msg_index = spec[0] - 1
|
||
|
|
# An empty string msgid isn't valid, emit a warning
|
||
|
|
if not messages[first_msg_index]:
|
||
|
|
filename = (getattr(fileobj, "name", None) or "(unknown)")
|
||
|
|
sys.stderr.write(
|
||
|
|
f"{filename}:{lineno}: warning: Empty msgid. It is reserved by GNU gettext: gettext(\"\") "
|
||
|
|
f"returns the header entry with meta information, not the empty string.\n",
|
||
|
|
)
|
||
|
|
return
|
||
|
|
|
||
|
|
translatable = tuple(translatable)
|
||
|
|
if len(translatable) == 1:
|
||
|
|
translatable = translatable[0]
|
||
|
|
|
||
|
|
return lineno, translatable, comments, context
|
||
|
|
|
||
|
|
|
||
|
|
def extract(
|
||
|
|
method: _ExtractionMethod,
|
||
|
|
fileobj: _FileObj,
|
||
|
|
keywords: Mapping[str, _Keyword] = DEFAULT_KEYWORDS,
|
||
|
|
comment_tags: Collection[str] = (),
|
||
|
|
options: Mapping[str, Any] | None = None,
|
||
|
|
strip_comment_tags: bool = False,
|
||
|
|
) -> Generator[_ExtractionResult, None, None]:
|
||
|
|
"""Extract messages from the given file-like object using the specified
|
||
|
|
extraction method.
|
||
|
|
|
||
|
|
This function returns tuples of the form ``(lineno, message, comments, context)``.
|
||
|
|
|
||
|
|
The implementation dispatches the actual extraction to plugins, based on the
|
||
|
|
value of the ``method`` parameter.
|
||
|
|
|
||
|
|
>>> source = b'''# foo module
|
||
|
|
... def run(argv):
|
||
|
|
... print(_('Hello, world!'))
|
||
|
|
... '''
|
||
|
|
|
||
|
|
>>> from io import BytesIO
|
||
|
|
>>> for message in extract('python', BytesIO(source)):
|
||
|
|
... print(message)
|
||
|
|
(3, u'Hello, world!', [], None)
|
||
|
|
|
||
|
|
:param method: an extraction method (a callable), or
|
||
|
|
a string specifying the extraction method (.e.g. "python");
|
||
|
|
if this is a simple name, the extraction function will be
|
||
|
|
looked up by entry point; if it is an explicit reference
|
||
|
|
to a function (of the form ``package.module:funcname`` or
|
||
|
|
``package.module.funcname``), the corresponding function
|
||
|
|
will be imported and used
|
||
|
|
:param fileobj: the file-like object the messages should be extracted from
|
||
|
|
:param keywords: a dictionary mapping keywords (i.e. names of functions
|
||
|
|
that should be recognized as translation functions) to
|
||
|
|
tuples that specify which of their arguments contain
|
||
|
|
localizable strings
|
||
|
|
:param comment_tags: a list of translator tags to search for and include
|
||
|
|
in the results
|
||
|
|
:param options: a dictionary of additional options (optional)
|
||
|
|
:param strip_comment_tags: a flag that if set to `True` causes all comment
|
||
|
|
tags to be removed from the collected comments.
|
||
|
|
:raise ValueError: if the extraction method is not registered
|
||
|
|
:returns: iterable of tuples of the form ``(lineno, message, comments, context)``
|
||
|
|
:rtype: Iterable[tuple[int, str|tuple[str], list[str], str|None]
|
||
|
|
"""
|
||
|
|
func = None
|
||
|
|
if callable(method):
|
||
|
|
func = method
|
||
|
|
elif ':' in method or '.' in method:
|
||
|
|
if ':' not in method:
|
||
|
|
lastdot = method.rfind('.')
|
||
|
|
module, attrname = method[:lastdot], method[lastdot + 1:]
|
||
|
|
else:
|
||
|
|
module, attrname = method.split(':', 1)
|
||
|
|
func = getattr(__import__(module, {}, {}, [attrname]), attrname)
|
||
|
|
else:
|
||
|
|
try:
|
||
|
|
from pkg_resources import working_set
|
||
|
|
except ImportError:
|
||
|
|
pass
|
||
|
|
else:
|
||
|
|
for entry_point in working_set.iter_entry_points(GROUP_NAME,
|
||
|
|
method):
|
||
|
|
func = entry_point.load(require=True)
|
||
|
|
break
|
||
|
|
if func is None:
|
||
|
|
# if pkg_resources is not available or no usable egg-info was found
|
||
|
|
# (see #230), we resort to looking up the builtin extractors
|
||
|
|
# directly
|
||
|
|
builtin = {
|
||
|
|
'ignore': extract_nothing,
|
||
|
|
'python': extract_python,
|
||
|
|
'javascript': extract_javascript,
|
||
|
|
}
|
||
|
|
func = builtin.get(method)
|
||
|
|
|
||
|
|
if func is None:
|
||
|
|
raise ValueError(f"Unknown extraction method {method!r}")
|
||
|
|
|
||
|
|
results = func(fileobj, keywords.keys(), comment_tags,
|
||
|
|
options=options or {})
|
||
|
|
|
||
|
|
for lineno, funcname, messages, comments in results:
|
||
|
|
if not isinstance(messages, (list, tuple)):
|
||
|
|
messages = [messages]
|
||
|
|
if not messages:
|
||
|
|
continue
|
||
|
|
|
||
|
|
specs = keywords[funcname] or None if funcname else None
|
||
|
|
# {None: x} may be collapsed into x for backwards compatibility.
|
||
|
|
if not isinstance(specs, dict):
|
||
|
|
specs = {None: specs}
|
||
|
|
|
||
|
|
if strip_comment_tags:
|
||
|
|
_strip_comment_tags(comments, comment_tags)
|
||
|
|
|
||
|
|
# None matches all arities.
|
||
|
|
for arity in (None, len(messages)):
|
||
|
|
try:
|
||
|
|
spec = specs[arity]
|
||
|
|
except KeyError:
|
||
|
|
continue
|
||
|
|
if spec is None:
|
||
|
|
spec = (1,)
|
||
|
|
result = _match_messages_against_spec(lineno, messages, comments, fileobj, spec)
|
||
|
|
if result is not None:
|
||
|
|
yield result
|
||
|
|
|
||
|
|
|
||
|
|
def extract_nothing(
|
||
|
|
fileobj: _FileObj,
|
||
|
|
keywords: Mapping[str, _Keyword],
|
||
|
|
comment_tags: Collection[str],
|
||
|
|
options: Mapping[str, Any],
|
||
|
|
) -> list[_ExtractionResult]:
|
||
|
|
"""Pseudo extractor that does not actually extract anything, but simply
|
||
|
|
returns an empty list.
|
||
|
|
"""
|
||
|
|
return []
|
||
|
|
|
||
|
|
|
||
|
|
def extract_python(
|
||
|
|
fileobj: IO[bytes],
|
||
|
|
keywords: Mapping[str, _Keyword],
|
||
|
|
comment_tags: Collection[str],
|
||
|
|
options: _PyOptions,
|
||
|
|
) -> Generator[_ExtractionResult, None, None]:
|
||
|
|
"""Extract messages from Python source code.
|
||
|
|
|
||
|
|
It returns an iterator yielding tuples in the following form ``(lineno,
|
||
|
|
funcname, message, comments)``.
|
||
|
|
|
||
|
|
:param fileobj: the seekable, file-like object the messages should be
|
||
|
|
extracted from
|
||
|
|
:param keywords: a list of keywords (i.e. function names) that should be
|
||
|
|
recognized as translation functions
|
||
|
|
:param comment_tags: a list of translator tags to search for and include
|
||
|
|
in the results
|
||
|
|
:param options: a dictionary of additional options (optional)
|
||
|
|
:rtype: ``iterator``
|
||
|
|
"""
|
||
|
|
funcname = lineno = message_lineno = None
|
||
|
|
call_stack = -1
|
||
|
|
buf = []
|
||
|
|
messages = []
|
||
|
|
translator_comments = []
|
||
|
|
in_def = in_translator_comments = False
|
||
|
|
comment_tag = None
|
||
|
|
|
||
|
|
encoding = parse_encoding(fileobj) or options.get('encoding', 'UTF-8')
|
||
|
|
future_flags = parse_future_flags(fileobj, encoding)
|
||
|
|
next_line = lambda: fileobj.readline().decode(encoding)
|
||
|
|
|
||
|
|
tokens = generate_tokens(next_line)
|
||
|
|
|
||
|
|
# Current prefix of a Python 3.12 (PEP 701) f-string, or None if we're not
|
||
|
|
# currently parsing one.
|
||
|
|
current_fstring_start = None
|
||
|
|
|
||
|
|
for tok, value, (lineno, _), _, _ in tokens:
|
||
|
|
if call_stack == -1 and tok == NAME and value in ('def', 'class'):
|
||
|
|
in_def = True
|
||
|
|
elif tok == OP and value == '(':
|
||
|
|
if in_def:
|
||
|
|
# Avoid false positives for declarations such as:
|
||
|
|
# def gettext(arg='message'):
|
||
|
|
in_def = False
|
||
|
|
continue
|
||
|
|
if funcname:
|
||
|
|
message_lineno = lineno
|
||
|
|
call_stack += 1
|
||
|
|
elif in_def and tok == OP and value == ':':
|
||
|
|
# End of a class definition without parens
|
||
|
|
in_def = False
|
||
|
|
continue
|
||
|
|
elif call_stack == -1 and tok == COMMENT:
|
||
|
|
# Strip the comment token from the line
|
||
|
|
value = value[1:].strip()
|
||
|
|
if in_translator_comments and \
|
||
|
|
translator_comments[-1][0] == lineno - 1:
|
||
|
|
# We're already inside a translator comment, continue appending
|
||
|
|
translator_comments.append((lineno, value))
|
||
|
|
continue
|
||
|
|
# If execution reaches this point, let's see if comment line
|
||
|
|
# starts with one of the comment tags
|
||
|
|
for comment_tag in comment_tags:
|
||
|
|
if value.startswith(comment_tag):
|
||
|
|
in_translator_comments = True
|
||
|
|
translator_comments.append((lineno, value))
|
||
|
|
break
|
||
|
|
elif funcname and call_stack == 0:
|
||
|
|
nested = (tok == NAME and value in keywords)
|
||
|
|
if (tok == OP and value == ')') or nested:
|
||
|
|
if buf:
|
||
|
|
messages.append(''.join(buf))
|
||
|
|
del buf[:]
|
||
|
|
else:
|
||
|
|
messages.append(None)
|
||
|
|
|
||
|
|
messages = tuple(messages) if len(messages) > 1 else messages[0]
|
||
|
|
# Comments don't apply unless they immediately
|
||
|
|
# precede the message
|
||
|
|
if translator_comments and \
|
||
|
|
translator_comments[-1][0] < message_lineno - 1:
|
||
|
|
translator_comments = []
|
||
|
|
|
||
|
|
yield (message_lineno, funcname, messages,
|
||
|
|
[comment[1] for comment in translator_comments])
|
||
|
|
|
||
|
|
funcname = lineno = message_lineno = None
|
||
|
|
call_stack = -1
|
||
|
|
messages = []
|
||
|
|
translator_comments = []
|
||
|
|
in_translator_comments = False
|
||
|
|
if nested:
|
||
|
|
funcname = value
|
||
|
|
elif tok == STRING:
|
||
|
|
val = _parse_python_string(value, encoding, future_flags)
|
||
|
|
if val is not None:
|
||
|
|
buf.append(val)
|
||
|
|
|
||
|
|
# Python 3.12+, see https://peps.python.org/pep-0701/#new-tokens
|
||
|
|
elif tok == FSTRING_START:
|
||
|
|
current_fstring_start = value
|
||
|
|
elif tok == FSTRING_MIDDLE:
|
||
|
|
if current_fstring_start is not None:
|
||
|
|
current_fstring_start += value
|
||
|
|
elif tok == FSTRING_END:
|
||
|
|
if current_fstring_start is not None:
|
||
|
|
fstring = current_fstring_start + value
|
||
|
|
val = _parse_python_string(fstring, encoding, future_flags)
|
||
|
|
if val is not None:
|
||
|
|
buf.append(val)
|
||
|
|
|
||
|
|
elif tok == OP and value == ',':
|
||
|
|
if buf:
|
||
|
|
messages.append(''.join(buf))
|
||
|
|
del buf[:]
|
||
|
|
else:
|
||
|
|
messages.append(None)
|
||
|
|
if translator_comments:
|
||
|
|
# We have translator comments, and since we're on a
|
||
|
|
# comma(,) user is allowed to break into a new line
|
||
|
|
# Let's increase the last comment's lineno in order
|
||
|
|
# for the comment to still be a valid one
|
||
|
|
old_lineno, old_comment = translator_comments.pop()
|
||
|
|
translator_comments.append((old_lineno + 1, old_comment))
|
||
|
|
elif call_stack > 0 and tok == OP and value == ')':
|
||
|
|
call_stack -= 1
|
||
|
|
elif funcname and call_stack == -1:
|
||
|
|
funcname = None
|
||
|
|
elif tok == NAME and value in keywords:
|
||
|
|
funcname = value
|
||
|
|
|
||
|
|
if (current_fstring_start is not None
|
||
|
|
and tok not in {FSTRING_START, FSTRING_MIDDLE}
|
||
|
|
):
|
||
|
|
# In Python 3.12, tokens other than FSTRING_* mean the
|
||
|
|
# f-string is dynamic, so we don't wan't to extract it.
|
||
|
|
# And if it's FSTRING_END, we've already handled it above.
|
||
|
|
# Let's forget that we're in an f-string.
|
||
|
|
current_fstring_start = None
|
||
|
|
|
||
|
|
|
||
|
|
def _parse_python_string(value: str, encoding: str, future_flags: int) -> str | None:
|
||
|
|
# Unwrap quotes in a safe manner, maintaining the string's encoding
|
||
|
|
# https://sourceforge.net/tracker/?func=detail&atid=355470&aid=617979&group_id=5470
|
||
|
|
code = compile(
|
||
|
|
f'# coding={str(encoding)}\n{value}',
|
||
|
|
'<string>',
|
||
|
|
'eval',
|
||
|
|
ast.PyCF_ONLY_AST | future_flags,
|
||
|
|
)
|
||
|
|
if isinstance(code, ast.Expression):
|
||
|
|
body = code.body
|
||
|
|
if isinstance(body, ast.Str):
|
||
|
|
return body.s
|
||
|
|
if isinstance(body, ast.JoinedStr): # f-string
|
||
|
|
if all(isinstance(node, ast.Str) for node in body.values):
|
||
|
|
return ''.join(node.s for node in body.values)
|
||
|
|
if all(isinstance(node, ast.Constant) for node in body.values):
|
||
|
|
return ''.join(str(node.value) for node in body.values)
|
||
|
|
# TODO: we could raise an error or warning when not all nodes are constants
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def extract_javascript(
|
||
|
|
fileobj: _FileObj,
|
||
|
|
keywords: Mapping[str, _Keyword],
|
||
|
|
comment_tags: Collection[str],
|
||
|
|
options: _JSOptions,
|
||
|
|
lineno: int = 1,
|
||
|
|
) -> Generator[_ExtractionResult, None, None]:
|
||
|
|
"""Extract messages from JavaScript source code.
|
||
|
|
|
||
|
|
:param fileobj: the seekable, file-like object the messages should be
|
||
|
|
extracted from
|
||
|
|
:param keywords: a list of keywords (i.e. function names) that should be
|
||
|
|
recognized as translation functions
|
||
|
|
:param comment_tags: a list of translator tags to search for and include
|
||
|
|
in the results
|
||
|
|
:param options: a dictionary of additional options (optional)
|
||
|
|
Supported options are:
|
||
|
|
* `jsx` -- set to false to disable JSX/E4X support.
|
||
|
|
* `template_string` -- if `True`, supports gettext(`key`)
|
||
|
|
* `parse_template_string` -- if `True` will parse the
|
||
|
|
contents of javascript
|
||
|
|
template strings.
|
||
|
|
:param lineno: line number offset (for parsing embedded fragments)
|
||
|
|
"""
|
||
|
|
from babel.messages.jslexer import Token, tokenize, unquote_string
|
||
|
|
funcname = message_lineno = None
|
||
|
|
messages = []
|
||
|
|
last_argument = None
|
||
|
|
translator_comments = []
|
||
|
|
concatenate_next = False
|
||
|
|
encoding = options.get('encoding', 'utf-8')
|
||
|
|
last_token = None
|
||
|
|
call_stack = -1
|
||
|
|
dotted = any('.' in kw for kw in keywords)
|
||
|
|
for token in tokenize(
|
||
|
|
fileobj.read().decode(encoding),
|
||
|
|
jsx=options.get("jsx", True),
|
||
|
|
template_string=options.get("template_string", True),
|
||
|
|
dotted=dotted,
|
||
|
|
lineno=lineno,
|
||
|
|
):
|
||
|
|
if ( # Turn keyword`foo` expressions into keyword("foo") calls:
|
||
|
|
funcname and # have a keyword...
|
||
|
|
(last_token and last_token.type == 'name') and # we've seen nothing after the keyword...
|
||
|
|
token.type == 'template_string' # this is a template string
|
||
|
|
):
|
||
|
|
message_lineno = token.lineno
|
||
|
|
messages = [unquote_string(token.value)]
|
||
|
|
call_stack = 0
|
||
|
|
token = Token('operator', ')', token.lineno)
|
||
|
|
|
||
|
|
if options.get('parse_template_string') and not funcname and token.type == 'template_string':
|
||
|
|
yield from parse_template_string(token.value, keywords, comment_tags, options, token.lineno)
|
||
|
|
|
||
|
|
elif token.type == 'operator' and token.value == '(':
|
||
|
|
if funcname:
|
||
|
|
message_lineno = token.lineno
|
||
|
|
call_stack += 1
|
||
|
|
|
||
|
|
elif call_stack == -1 and token.type == 'linecomment':
|
||
|
|
value = token.value[2:].strip()
|
||
|
|
if translator_comments and \
|
||
|
|
translator_comments[-1][0] == token.lineno - 1:
|
||
|
|
translator_comments.append((token.lineno, value))
|
||
|
|
continue
|
||
|
|
|
||
|
|
for comment_tag in comment_tags:
|
||
|
|
if value.startswith(comment_tag):
|
||
|
|
translator_comments.append((token.lineno, value.strip()))
|
||
|
|
break
|
||
|
|
|
||
|
|
elif token.type == 'multilinecomment':
|
||
|
|
# only one multi-line comment may precede a translation
|
||
|
|
translator_comments = []
|
||
|
|
value = token.value[2:-2].strip()
|
||
|
|
for comment_tag in comment_tags:
|
||
|
|
if value.startswith(comment_tag):
|
||
|
|
lines = value.splitlines()
|
||
|
|
if lines:
|
||
|
|
lines[0] = lines[0].strip()
|
||
|
|
lines[1:] = dedent('\n'.join(lines[1:])).splitlines()
|
||
|
|
for offset, line in enumerate(lines):
|
||
|
|
translator_comments.append((token.lineno + offset,
|
||
|
|
line))
|
||
|
|
break
|
||
|
|
|
||
|
|
elif funcname and call_stack == 0:
|
||
|
|
if token.type == 'operator' and token.value == ')':
|
||
|
|
if last_argument is not None:
|
||
|
|
messages.append(last_argument)
|
||
|
|
if len(messages) > 1:
|
||
|
|
messages = tuple(messages)
|
||
|
|
elif messages:
|
||
|
|
messages = messages[0]
|
||
|
|
else:
|
||
|
|
messages = None
|
||
|
|
|
||
|
|
# Comments don't apply unless they immediately precede the
|
||
|
|
# message
|
||
|
|
if translator_comments and \
|
||
|
|
translator_comments[-1][0] < message_lineno - 1:
|
||
|
|
translator_comments = []
|
||
|
|
|
||
|
|
if messages is not None:
|
||
|
|
yield (message_lineno, funcname, messages,
|
||
|
|
[comment[1] for comment in translator_comments])
|
||
|
|
|
||
|
|
funcname = message_lineno = last_argument = None
|
||
|
|
concatenate_next = False
|
||
|
|
translator_comments = []
|
||
|
|
messages = []
|
||
|
|
call_stack = -1
|
||
|
|
|
||
|
|
elif token.type in ('string', 'template_string'):
|
||
|
|
new_value = unquote_string(token.value)
|
||
|
|
if concatenate_next:
|
||
|
|
last_argument = (last_argument or '') + new_value
|
||
|
|
concatenate_next = False
|
||
|
|
else:
|
||
|
|
last_argument = new_value
|
||
|
|
|
||
|
|
elif token.type == 'operator':
|
||
|
|
if token.value == ',':
|
||
|
|
if last_argument is not None:
|
||
|
|
messages.append(last_argument)
|
||
|
|
last_argument = None
|
||
|
|
else:
|
||
|
|
messages.append(None)
|
||
|
|
concatenate_next = False
|
||
|
|
elif token.value == '+':
|
||
|
|
concatenate_next = True
|
||
|
|
|
||
|
|
elif call_stack > 0 and token.type == 'operator' \
|
||
|
|
and token.value == ')':
|
||
|
|
call_stack -= 1
|
||
|
|
|
||
|
|
elif funcname and call_stack == -1:
|
||
|
|
funcname = None
|
||
|
|
|
||
|
|
elif call_stack == -1 and token.type == 'name' and \
|
||
|
|
token.value in keywords and \
|
||
|
|
(last_token is None or last_token.type != 'name' or
|
||
|
|
last_token.value != 'function'):
|
||
|
|
funcname = token.value
|
||
|
|
|
||
|
|
last_token = token
|
||
|
|
|
||
|
|
|
||
|
|
def parse_template_string(
|
||
|
|
template_string: str,
|
||
|
|
keywords: Mapping[str, _Keyword],
|
||
|
|
comment_tags: Collection[str],
|
||
|
|
options: _JSOptions,
|
||
|
|
lineno: int = 1,
|
||
|
|
) -> Generator[_ExtractionResult, None, None]:
|
||
|
|
"""Parse JavaScript template string.
|
||
|
|
|
||
|
|
:param template_string: the template string to be parsed
|
||
|
|
:param keywords: a list of keywords (i.e. function names) that should be
|
||
|
|
recognized as translation functions
|
||
|
|
:param comment_tags: a list of translator tags to search for and include
|
||
|
|
in the results
|
||
|
|
:param options: a dictionary of additional options (optional)
|
||
|
|
:param lineno: starting line number (optional)
|
||
|
|
"""
|
||
|
|
from babel.messages.jslexer import line_re
|
||
|
|
prev_character = None
|
||
|
|
level = 0
|
||
|
|
inside_str = False
|
||
|
|
expression_contents = ''
|
||
|
|
for character in template_string[1:-1]:
|
||
|
|
if not inside_str and character in ('"', "'", '`'):
|
||
|
|
inside_str = character
|
||
|
|
elif inside_str == character and prev_character != r'\\':
|
||
|
|
inside_str = False
|
||
|
|
if level:
|
||
|
|
expression_contents += character
|
||
|
|
if not inside_str:
|
||
|
|
if character == '{' and prev_character == '$':
|
||
|
|
level += 1
|
||
|
|
elif level and character == '}':
|
||
|
|
level -= 1
|
||
|
|
if level == 0 and expression_contents:
|
||
|
|
expression_contents = expression_contents[0:-1]
|
||
|
|
fake_file_obj = io.BytesIO(expression_contents.encode())
|
||
|
|
yield from extract_javascript(fake_file_obj, keywords, comment_tags, options, lineno)
|
||
|
|
lineno += len(line_re.findall(expression_contents))
|
||
|
|
expression_contents = ''
|
||
|
|
prev_character = character
|