343 lines
11 KiB
Python
343 lines
11 KiB
Python
|
# Generated content DO NOT EDIT
|
||
|
|
class PostProcessor:
|
||
|
|
"""
|
||
|
|
Base class for all post-processors
|
||
|
|
|
||
|
|
This class is not supposed to be instantiated directly. Instead, any implementation of
|
||
|
|
a PostProcessor will return an instance of this class when instantiated.
|
||
|
|
"""
|
||
|
|
def num_special_tokens_to_add(self, is_pair):
|
||
|
|
"""
|
||
|
|
Return the number of special tokens that would be added for single/pair sentences.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
is_pair (:obj:`bool`):
|
||
|
|
Whether the input would be a pair of sequences
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
:obj:`int`: The number of tokens to add
|
||
|
|
"""
|
||
|
|
pass
|
||
|
|
|
||
|
|
def process(self, encoding, pair=None, add_special_tokens=True):
|
||
|
|
"""
|
||
|
|
Post-process the given encodings, generating the final one
|
||
|
|
|
||
|
|
Args:
|
||
|
|
encoding (:class:`~tokenizers.Encoding`):
|
||
|
|
The encoding for the first sequence
|
||
|
|
|
||
|
|
pair (:class:`~tokenizers.Encoding`, `optional`):
|
||
|
|
The encoding for the pair sequence
|
||
|
|
|
||
|
|
add_special_tokens (:obj:`bool`):
|
||
|
|
Whether to add the special tokens
|
||
|
|
|
||
|
|
Return:
|
||
|
|
:class:`~tokenizers.Encoding`: The final encoding
|
||
|
|
"""
|
||
|
|
pass
|
||
|
|
|
||
|
|
class BertProcessing(PostProcessor):
|
||
|
|
"""
|
||
|
|
This post-processor takes care of adding the special tokens needed by
|
||
|
|
a Bert model:
|
||
|
|
|
||
|
|
- a SEP token
|
||
|
|
- a CLS token
|
||
|
|
|
||
|
|
Args:
|
||
|
|
sep (:obj:`Tuple[str, int]`):
|
||
|
|
A tuple with the string representation of the SEP token, and its id
|
||
|
|
|
||
|
|
cls (:obj:`Tuple[str, int]`):
|
||
|
|
A tuple with the string representation of the CLS token, and its id
|
||
|
|
"""
|
||
|
|
def __init__(self, sep, cls):
|
||
|
|
pass
|
||
|
|
|
||
|
|
def num_special_tokens_to_add(self, is_pair):
|
||
|
|
"""
|
||
|
|
Return the number of special tokens that would be added for single/pair sentences.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
is_pair (:obj:`bool`):
|
||
|
|
Whether the input would be a pair of sequences
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
:obj:`int`: The number of tokens to add
|
||
|
|
"""
|
||
|
|
pass
|
||
|
|
|
||
|
|
def process(self, encoding, pair=None, add_special_tokens=True):
|
||
|
|
"""
|
||
|
|
Post-process the given encodings, generating the final one
|
||
|
|
|
||
|
|
Args:
|
||
|
|
encoding (:class:`~tokenizers.Encoding`):
|
||
|
|
The encoding for the first sequence
|
||
|
|
|
||
|
|
pair (:class:`~tokenizers.Encoding`, `optional`):
|
||
|
|
The encoding for the pair sequence
|
||
|
|
|
||
|
|
add_special_tokens (:obj:`bool`):
|
||
|
|
Whether to add the special tokens
|
||
|
|
|
||
|
|
Return:
|
||
|
|
:class:`~tokenizers.Encoding`: The final encoding
|
||
|
|
"""
|
||
|
|
pass
|
||
|
|
|
||
|
|
class ByteLevel(PostProcessor):
|
||
|
|
"""
|
||
|
|
This post-processor takes care of trimming the offsets.
|
||
|
|
|
||
|
|
By default, the ByteLevel BPE might include whitespaces in the produced tokens. If you don't
|
||
|
|
want the offsets to include these whitespaces, then this PostProcessor must be used.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
trim_offsets (:obj:`bool`):
|
||
|
|
Whether to trim the whitespaces from the produced offsets.
|
||
|
|
"""
|
||
|
|
def __init__(self, trim_offsets=True):
|
||
|
|
pass
|
||
|
|
|
||
|
|
def num_special_tokens_to_add(self, is_pair):
|
||
|
|
"""
|
||
|
|
Return the number of special tokens that would be added for single/pair sentences.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
is_pair (:obj:`bool`):
|
||
|
|
Whether the input would be a pair of sequences
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
:obj:`int`: The number of tokens to add
|
||
|
|
"""
|
||
|
|
pass
|
||
|
|
|
||
|
|
def process(self, encoding, pair=None, add_special_tokens=True):
|
||
|
|
"""
|
||
|
|
Post-process the given encodings, generating the final one
|
||
|
|
|
||
|
|
Args:
|
||
|
|
encoding (:class:`~tokenizers.Encoding`):
|
||
|
|
The encoding for the first sequence
|
||
|
|
|
||
|
|
pair (:class:`~tokenizers.Encoding`, `optional`):
|
||
|
|
The encoding for the pair sequence
|
||
|
|
|
||
|
|
add_special_tokens (:obj:`bool`):
|
||
|
|
Whether to add the special tokens
|
||
|
|
|
||
|
|
Return:
|
||
|
|
:class:`~tokenizers.Encoding`: The final encoding
|
||
|
|
"""
|
||
|
|
pass
|
||
|
|
|
||
|
|
class RobertaProcessing(PostProcessor):
|
||
|
|
"""
|
||
|
|
This post-processor takes care of adding the special tokens needed by
|
||
|
|
a Roberta model:
|
||
|
|
|
||
|
|
- a SEP token
|
||
|
|
- a CLS token
|
||
|
|
|
||
|
|
It also takes care of trimming the offsets.
|
||
|
|
By default, the ByteLevel BPE might include whitespaces in the produced tokens. If you don't
|
||
|
|
want the offsets to include these whitespaces, then this PostProcessor should be initialized
|
||
|
|
with :obj:`trim_offsets=True`
|
||
|
|
|
||
|
|
Args:
|
||
|
|
sep (:obj:`Tuple[str, int]`):
|
||
|
|
A tuple with the string representation of the SEP token, and its id
|
||
|
|
|
||
|
|
cls (:obj:`Tuple[str, int]`):
|
||
|
|
A tuple with the string representation of the CLS token, and its id
|
||
|
|
|
||
|
|
trim_offsets (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||
|
|
Whether to trim the whitespaces from the produced offsets.
|
||
|
|
|
||
|
|
add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`):
|
||
|
|
Whether the add_prefix_space option was enabled during pre-tokenization. This
|
||
|
|
is relevant because it defines the way the offsets are trimmed out.
|
||
|
|
"""
|
||
|
|
def __init__(self, sep, cls, trim_offsets=True, add_prefix_space=True):
|
||
|
|
pass
|
||
|
|
|
||
|
|
def num_special_tokens_to_add(self, is_pair):
|
||
|
|
"""
|
||
|
|
Return the number of special tokens that would be added for single/pair sentences.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
is_pair (:obj:`bool`):
|
||
|
|
Whether the input would be a pair of sequences
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
:obj:`int`: The number of tokens to add
|
||
|
|
"""
|
||
|
|
pass
|
||
|
|
|
||
|
|
def process(self, encoding, pair=None, add_special_tokens=True):
|
||
|
|
"""
|
||
|
|
Post-process the given encodings, generating the final one
|
||
|
|
|
||
|
|
Args:
|
||
|
|
encoding (:class:`~tokenizers.Encoding`):
|
||
|
|
The encoding for the first sequence
|
||
|
|
|
||
|
|
pair (:class:`~tokenizers.Encoding`, `optional`):
|
||
|
|
The encoding for the pair sequence
|
||
|
|
|
||
|
|
add_special_tokens (:obj:`bool`):
|
||
|
|
Whether to add the special tokens
|
||
|
|
|
||
|
|
Return:
|
||
|
|
:class:`~tokenizers.Encoding`: The final encoding
|
||
|
|
"""
|
||
|
|
pass
|
||
|
|
|
||
|
|
class Sequence(PostProcessor):
|
||
|
|
"""
|
||
|
|
Sequence Processor
|
||
|
|
|
||
|
|
Args:
|
||
|
|
processors (:obj:`List[PostProcessor]`)
|
||
|
|
The processors that need to be chained
|
||
|
|
"""
|
||
|
|
def __init__(self, processors):
|
||
|
|
pass
|
||
|
|
|
||
|
|
def num_special_tokens_to_add(self, is_pair):
|
||
|
|
"""
|
||
|
|
Return the number of special tokens that would be added for single/pair sentences.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
is_pair (:obj:`bool`):
|
||
|
|
Whether the input would be a pair of sequences
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
:obj:`int`: The number of tokens to add
|
||
|
|
"""
|
||
|
|
pass
|
||
|
|
|
||
|
|
def process(self, encoding, pair=None, add_special_tokens=True):
|
||
|
|
"""
|
||
|
|
Post-process the given encodings, generating the final one
|
||
|
|
|
||
|
|
Args:
|
||
|
|
encoding (:class:`~tokenizers.Encoding`):
|
||
|
|
The encoding for the first sequence
|
||
|
|
|
||
|
|
pair (:class:`~tokenizers.Encoding`, `optional`):
|
||
|
|
The encoding for the pair sequence
|
||
|
|
|
||
|
|
add_special_tokens (:obj:`bool`):
|
||
|
|
Whether to add the special tokens
|
||
|
|
|
||
|
|
Return:
|
||
|
|
:class:`~tokenizers.Encoding`: The final encoding
|
||
|
|
"""
|
||
|
|
pass
|
||
|
|
|
||
|
|
class TemplateProcessing(PostProcessor):
|
||
|
|
"""
|
||
|
|
Provides a way to specify templates in order to add the special tokens to each
|
||
|
|
input sequence as relevant.
|
||
|
|
|
||
|
|
Let's take :obj:`BERT` tokenizer as an example. It uses two special tokens, used to
|
||
|
|
delimitate each sequence. :obj:`[CLS]` is always used at the beginning of the first
|
||
|
|
sequence, and :obj:`[SEP]` is added at the end of both the first, and the pair
|
||
|
|
sequences. The final result looks like this:
|
||
|
|
|
||
|
|
- Single sequence: :obj:`[CLS] Hello there [SEP]`
|
||
|
|
- Pair sequences: :obj:`[CLS] My name is Anthony [SEP] What is my name? [SEP]`
|
||
|
|
|
||
|
|
With the type ids as following::
|
||
|
|
|
||
|
|
[CLS] ... [SEP] ... [SEP]
|
||
|
|
0 0 0 1 1
|
||
|
|
|
||
|
|
You can achieve such behavior using a TemplateProcessing::
|
||
|
|
|
||
|
|
TemplateProcessing(
|
||
|
|
single="[CLS] $0 [SEP]",
|
||
|
|
pair="[CLS] $A [SEP] $B:1 [SEP]:1",
|
||
|
|
special_tokens=[("[CLS]", 1), ("[SEP]", 0)],
|
||
|
|
)
|
||
|
|
|
||
|
|
In this example, each input sequence is identified using a ``$`` construct. This identifier
|
||
|
|
lets us specify each input sequence, and the type_id to use. When nothing is specified,
|
||
|
|
it uses the default values. Here are the different ways to specify it:
|
||
|
|
|
||
|
|
- Specifying the sequence, with default ``type_id == 0``: ``$A`` or ``$B``
|
||
|
|
- Specifying the `type_id` with default ``sequence == A``: ``$0``, ``$1``, ``$2``, ...
|
||
|
|
- Specifying both: ``$A:0``, ``$B:1``, ...
|
||
|
|
|
||
|
|
The same construct is used for special tokens: ``<identifier>(:<type_id>)?``.
|
||
|
|
|
||
|
|
**Warning**: You must ensure that you are giving the correct tokens/ids as these
|
||
|
|
will be added to the Encoding without any further check. If the given ids correspond
|
||
|
|
to something totally different in a `Tokenizer` using this `PostProcessor`, it
|
||
|
|
might lead to unexpected results.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
single (:obj:`Template`):
|
||
|
|
The template used for single sequences
|
||
|
|
|
||
|
|
pair (:obj:`Template`):
|
||
|
|
The template used when both sequences are specified
|
||
|
|
|
||
|
|
special_tokens (:obj:`Tokens`):
|
||
|
|
The list of special tokens used in each sequences
|
||
|
|
|
||
|
|
Types:
|
||
|
|
|
||
|
|
Template (:obj:`str` or :obj:`List`):
|
||
|
|
- If a :obj:`str` is provided, the whitespace is used as delimiter between tokens
|
||
|
|
- If a :obj:`List[str]` is provided, a list of tokens
|
||
|
|
|
||
|
|
Tokens (:obj:`List[Union[Tuple[int, str], Tuple[str, int], dict]]`):
|
||
|
|
- A :obj:`Tuple` with both a token and its associated ID, in any order
|
||
|
|
- A :obj:`dict` with the following keys:
|
||
|
|
- "id": :obj:`str` => The special token id, as specified in the Template
|
||
|
|
- "ids": :obj:`List[int]` => The associated IDs
|
||
|
|
- "tokens": :obj:`List[str]` => The associated tokens
|
||
|
|
|
||
|
|
The given dict expects the provided :obj:`ids` and :obj:`tokens` lists to have
|
||
|
|
the same length.
|
||
|
|
"""
|
||
|
|
def __init__(self, single, pair, special_tokens):
|
||
|
|
pass
|
||
|
|
|
||
|
|
def num_special_tokens_to_add(self, is_pair):
|
||
|
|
"""
|
||
|
|
Return the number of special tokens that would be added for single/pair sentences.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
is_pair (:obj:`bool`):
|
||
|
|
Whether the input would be a pair of sequences
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
:obj:`int`: The number of tokens to add
|
||
|
|
"""
|
||
|
|
pass
|
||
|
|
|
||
|
|
def process(self, encoding, pair=None, add_special_tokens=True):
|
||
|
|
"""
|
||
|
|
Post-process the given encodings, generating the final one
|
||
|
|
|
||
|
|
Args:
|
||
|
|
encoding (:class:`~tokenizers.Encoding`):
|
||
|
|
The encoding for the first sequence
|
||
|
|
|
||
|
|
pair (:class:`~tokenizers.Encoding`, `optional`):
|
||
|
|
The encoding for the pair sequence
|
||
|
|
|
||
|
|
add_special_tokens (:obj:`bool`):
|
||
|
|
Whether to add the special tokens
|
||
|
|
|
||
|
|
Return:
|
||
|
|
:class:`~tokenizers.Encoding`: The final encoding
|
||
|
|
"""
|
||
|
|
pass
|