60 lines
1.4 KiB
Python
60 lines
1.4 KiB
Python
|
# These exceptions are used to add NORM values based on a token's ORTH value.
|
|||
|
|
# Individual languages can also add their own exceptions and overwrite them -
|
|||
|
|
# for example, British vs. American spelling in English.
|
|||
|
|
|
|||
|
|
# Norms are only set if no alternative is provided in the tokenizer exceptions.
|
|||
|
|
# Note that this does not change any other token attributes. Its main purpose
|
|||
|
|
# is to normalise the word representations so that equivalent tokens receive
|
|||
|
|
# similar representations. For example: $ and € are very different, but they're
|
|||
|
|
# both currency symbols. By normalising currency symbols to $, all symbols are
|
|||
|
|
# seen as similar, no matter how common they are in the training data.
|
|||
|
|
|
|||
|
|
|
|||
|
|
BASE_NORMS = {
|
|||
|
|
"'s": "'s",
|
|||
|
|
"'S": "'s",
|
|||
|
|
"’s": "'s",
|
|||
|
|
"’S": "'s",
|
|||
|
|
"’": "'",
|
|||
|
|
"‘": "'",
|
|||
|
|
"´": "'",
|
|||
|
|
"`": "'",
|
|||
|
|
"”": '"',
|
|||
|
|
"“": '"',
|
|||
|
|
"''": '"',
|
|||
|
|
"``": '"',
|
|||
|
|
"´´": '"',
|
|||
|
|
"„": '"',
|
|||
|
|
"»": '"',
|
|||
|
|
"«": '"',
|
|||
|
|
"‘‘": '"',
|
|||
|
|
"’’": '"',
|
|||
|
|
"？": "?",
|
|||
|
|
"！": "!",
|
|||
|
|
"，": ",",
|
|||
|
|
"；": ";",
|
|||
|
|
"：": ":",
|
|||
|
|
"。": ".",
|
|||
|
|
"।": ".",
|
|||
|
|
"…": "...",
|
|||
|
|
"—": "-",
|
|||
|
|
"–": "-",
|
|||
|
|
"--": "-",
|
|||
|
|
"---": "-",
|
|||
|
|
"——": "-",
|
|||
|
|
"€": "$",
|
|||
|
|
"£": "$",
|
|||
|
|
"¥": "$",
|
|||
|
|
"฿": "$",
|
|||
|
|
"US$": "$",
|
|||
|
|
"C$": "$",
|
|||
|
|
"A$": "$",
|
|||
|
|
"₺": "$",
|
|||
|
|
"₹": "$",
|
|||
|
|
"৳": "$",
|
|||
|
|
"₩": "$",
|
|||
|
|
"Mex$": "$",
|
|||
|
|
"₣": "$",
|
|||
|
|
"E£": "$",
|
|||
|
|
}
|