844 lines
30 KiB
Python
844 lines
30 KiB
Python
|
import os
|
|||
|
|
import re
|
|||
|
|
import textwrap
|
|||
|
|
from functools import cached_property
|
|||
|
|
|
|||
|
|
import pypinyin
|
|||
|
|
import torch
|
|||
|
|
from hangul_romanize import Transliter
|
|||
|
|
from hangul_romanize.rule import academic
|
|||
|
|
from num2words import num2words
|
|||
|
|
from spacy.lang.ar import Arabic
|
|||
|
|
from spacy.lang.en import English
|
|||
|
|
from spacy.lang.es import Spanish
|
|||
|
|
from spacy.lang.ja import Japanese
|
|||
|
|
from spacy.lang.zh import Chinese
|
|||
|
|
from tokenizers import Tokenizer
|
|||
|
|
|
|||
|
|
from TTS.tts.layers.xtts.zh_num2words import TextNorm as zh_num2words
|
|||
|
|
|
|||
|
|
|
|||
|
|
def get_spacy_lang(lang):
|
|||
|
|
if lang == "zh":
|
|||
|
|
return Chinese()
|
|||
|
|
elif lang == "ja":
|
|||
|
|
return Japanese()
|
|||
|
|
elif lang == "ar":
|
|||
|
|
return Arabic()
|
|||
|
|
elif lang == "es":
|
|||
|
|
return Spanish()
|
|||
|
|
else:
|
|||
|
|
# For most languages, Enlish does the job
|
|||
|
|
return English()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def split_sentence(text, lang, text_split_length=250):
|
|||
|
|
"""Preprocess the input text"""
|
|||
|
|
text_splits = []
|
|||
|
|
if text_split_length is not None and len(text) >= text_split_length:
|
|||
|
|
text_splits.append("")
|
|||
|
|
nlp = get_spacy_lang(lang)
|
|||
|
|
nlp.add_pipe("sentencizer")
|
|||
|
|
doc = nlp(text)
|
|||
|
|
for sentence in doc.sents:
|
|||
|
|
if len(text_splits[-1]) + len(str(sentence)) <= text_split_length:
|
|||
|
|
# if the last sentence + the current sentence is less than the text_split_length
|
|||
|
|
# then add the current sentence to the last sentence
|
|||
|
|
text_splits[-1] += " " + str(sentence)
|
|||
|
|
text_splits[-1] = text_splits[-1].lstrip()
|
|||
|
|
elif len(str(sentence)) > text_split_length:
|
|||
|
|
# if the current sentence is greater than the text_split_length
|
|||
|
|
for line in textwrap.wrap(
|
|||
|
|
str(sentence),
|
|||
|
|
width=text_split_length,
|
|||
|
|
drop_whitespace=True,
|
|||
|
|
break_on_hyphens=False,
|
|||
|
|
tabsize=1,
|
|||
|
|
):
|
|||
|
|
text_splits.append(str(line))
|
|||
|
|
else:
|
|||
|
|
text_splits.append(str(sentence))
|
|||
|
|
|
|||
|
|
if len(text_splits) > 1:
|
|||
|
|
if text_splits[0] == "":
|
|||
|
|
del text_splits[0]
|
|||
|
|
else:
|
|||
|
|
text_splits = [text.lstrip()]
|
|||
|
|
|
|||
|
|
return text_splits
|
|||
|
|
|
|||
|
|
|
|||
|
|
_whitespace_re = re.compile(r"\s+")
|
|||
|
|
|
|||
|
|
# List of (regular expression, replacement) pairs for abbreviations:
|
|||
|
|
_abbreviations = {
|
|||
|
|
"en": [
|
|||
|
|
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("mrs", "misess"),
|
|||
|
|
("mr", "mister"),
|
|||
|
|
("dr", "doctor"),
|
|||
|
|
("st", "saint"),
|
|||
|
|
("co", "company"),
|
|||
|
|
("jr", "junior"),
|
|||
|
|
("maj", "major"),
|
|||
|
|
("gen", "general"),
|
|||
|
|
("drs", "doctors"),
|
|||
|
|
("rev", "reverend"),
|
|||
|
|
("lt", "lieutenant"),
|
|||
|
|
("hon", "honorable"),
|
|||
|
|
("sgt", "sergeant"),
|
|||
|
|
("capt", "captain"),
|
|||
|
|
("esq", "esquire"),
|
|||
|
|
("ltd", "limited"),
|
|||
|
|
("col", "colonel"),
|
|||
|
|
("ft", "fort"),
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"es": [
|
|||
|
|
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("sra", "señora"),
|
|||
|
|
("sr", "señor"),
|
|||
|
|
("dr", "doctor"),
|
|||
|
|
("dra", "doctora"),
|
|||
|
|
("st", "santo"),
|
|||
|
|
("co", "compañía"),
|
|||
|
|
("jr", "junior"),
|
|||
|
|
("ltd", "limitada"),
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"fr": [
|
|||
|
|
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("mme", "madame"),
|
|||
|
|
("mr", "monsieur"),
|
|||
|
|
("dr", "docteur"),
|
|||
|
|
("st", "saint"),
|
|||
|
|
("co", "compagnie"),
|
|||
|
|
("jr", "junior"),
|
|||
|
|
("ltd", "limitée"),
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"de": [
|
|||
|
|
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("fr", "frau"),
|
|||
|
|
("dr", "doktor"),
|
|||
|
|
("st", "sankt"),
|
|||
|
|
("co", "firma"),
|
|||
|
|
("jr", "junior"),
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"pt": [
|
|||
|
|
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("sra", "senhora"),
|
|||
|
|
("sr", "senhor"),
|
|||
|
|
("dr", "doutor"),
|
|||
|
|
("dra", "doutora"),
|
|||
|
|
("st", "santo"),
|
|||
|
|
("co", "companhia"),
|
|||
|
|
("jr", "júnior"),
|
|||
|
|
("ltd", "limitada"),
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"it": [
|
|||
|
|
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
# ("sig.ra", "signora"),
|
|||
|
|
("sig", "signore"),
|
|||
|
|
("dr", "dottore"),
|
|||
|
|
("st", "santo"),
|
|||
|
|
("co", "compagnia"),
|
|||
|
|
("jr", "junior"),
|
|||
|
|
("ltd", "limitata"),
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"pl": [
|
|||
|
|
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("p", "pani"),
|
|||
|
|
("m", "pan"),
|
|||
|
|
("dr", "doktor"),
|
|||
|
|
("sw", "święty"),
|
|||
|
|
("jr", "junior"),
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"ar": [
|
|||
|
|
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
# There are not many common abbreviations in Arabic as in English.
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"zh": [
|
|||
|
|
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
# Chinese doesn't typically use abbreviations in the same way as Latin-based scripts.
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"cs": [
|
|||
|
|
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("dr", "doktor"), # doctor
|
|||
|
|
("ing", "inženýr"), # engineer
|
|||
|
|
("p", "pan"), # Could also map to pani for woman but no easy way to do it
|
|||
|
|
# Other abbreviations would be specialized and not as common.
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"ru": [
|
|||
|
|
(re.compile("\\b%s\\b" % x[0], re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("г-жа", "госпожа"), # Mrs.
|
|||
|
|
("г-н", "господин"), # Mr.
|
|||
|
|
("д-р", "доктор"), # doctor
|
|||
|
|
# Other abbreviations are less common or specialized.
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"nl": [
|
|||
|
|
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("dhr", "de heer"), # Mr.
|
|||
|
|
("mevr", "mevrouw"), # Mrs.
|
|||
|
|
("dr", "dokter"), # doctor
|
|||
|
|
("jhr", "jonkheer"), # young lord or nobleman
|
|||
|
|
# Dutch uses more abbreviations, but these are the most common ones.
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"tr": [
|
|||
|
|
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("b", "bay"), # Mr.
|
|||
|
|
("byk", "büyük"), # büyük
|
|||
|
|
("dr", "doktor"), # doctor
|
|||
|
|
# Add other Turkish abbreviations here if needed.
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"hu": [
|
|||
|
|
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("dr", "doktor"), # doctor
|
|||
|
|
("b", "bácsi"), # Mr.
|
|||
|
|
("nőv", "nővér"), # nurse
|
|||
|
|
# Add other Hungarian abbreviations here if needed.
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"ko": [
|
|||
|
|
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
# Korean doesn't typically use abbreviations in the same way as Latin-based scripts.
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def expand_abbreviations_multilingual(text, lang="en"):
|
|||
|
|
for regex, replacement in _abbreviations[lang]:
|
|||
|
|
text = re.sub(regex, replacement, text)
|
|||
|
|
return text
|
|||
|
|
|
|||
|
|
|
|||
|
|
_symbols_multilingual = {
|
|||
|
|
"en": [
|
|||
|
|
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("&", " and "),
|
|||
|
|
("@", " at "),
|
|||
|
|
("%", " percent "),
|
|||
|
|
("#", " hash "),
|
|||
|
|
("$", " dollar "),
|
|||
|
|
("£", " pound "),
|
|||
|
|
("°", " degree "),
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"es": [
|
|||
|
|
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("&", " y "),
|
|||
|
|
("@", " arroba "),
|
|||
|
|
("%", " por ciento "),
|
|||
|
|
("#", " numeral "),
|
|||
|
|
("$", " dolar "),
|
|||
|
|
("£", " libra "),
|
|||
|
|
("°", " grados "),
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"fr": [
|
|||
|
|
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("&", " et "),
|
|||
|
|
("@", " arobase "),
|
|||
|
|
("%", " pour cent "),
|
|||
|
|
("#", " dièse "),
|
|||
|
|
("$", " dollar "),
|
|||
|
|
("£", " livre "),
|
|||
|
|
("°", " degrés "),
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"de": [
|
|||
|
|
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("&", " und "),
|
|||
|
|
("@", " at "),
|
|||
|
|
("%", " prozent "),
|
|||
|
|
("#", " raute "),
|
|||
|
|
("$", " dollar "),
|
|||
|
|
("£", " pfund "),
|
|||
|
|
("°", " grad "),
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"pt": [
|
|||
|
|
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("&", " e "),
|
|||
|
|
("@", " arroba "),
|
|||
|
|
("%", " por cento "),
|
|||
|
|
("#", " cardinal "),
|
|||
|
|
("$", " dólar "),
|
|||
|
|
("£", " libra "),
|
|||
|
|
("°", " graus "),
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"it": [
|
|||
|
|
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("&", " e "),
|
|||
|
|
("@", " chiocciola "),
|
|||
|
|
("%", " per cento "),
|
|||
|
|
("#", " cancelletto "),
|
|||
|
|
("$", " dollaro "),
|
|||
|
|
("£", " sterlina "),
|
|||
|
|
("°", " gradi "),
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"pl": [
|
|||
|
|
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("&", " i "),
|
|||
|
|
("@", " małpa "),
|
|||
|
|
("%", " procent "),
|
|||
|
|
("#", " krzyżyk "),
|
|||
|
|
("$", " dolar "),
|
|||
|
|
("£", " funt "),
|
|||
|
|
("°", " stopnie "),
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"ar": [
|
|||
|
|
# Arabic
|
|||
|
|
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("&", " و "),
|
|||
|
|
("@", " على "),
|
|||
|
|
("%", " في المئة "),
|
|||
|
|
("#", " رقم "),
|
|||
|
|
("$", " دولار "),
|
|||
|
|
("£", " جنيه "),
|
|||
|
|
("°", " درجة "),
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"zh": [
|
|||
|
|
# Chinese
|
|||
|
|
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("&", " 和 "),
|
|||
|
|
("@", " 在 "),
|
|||
|
|
("%", " 百分之 "),
|
|||
|
|
("#", " 号 "),
|
|||
|
|
("$", " 美元 "),
|
|||
|
|
("£", " 英镑 "),
|
|||
|
|
("°", " 度 "),
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"cs": [
|
|||
|
|
# Czech
|
|||
|
|
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("&", " a "),
|
|||
|
|
("@", " na "),
|
|||
|
|
("%", " procento "),
|
|||
|
|
("#", " křížek "),
|
|||
|
|
("$", " dolar "),
|
|||
|
|
("£", " libra "),
|
|||
|
|
("°", " stupně "),
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"ru": [
|
|||
|
|
# Russian
|
|||
|
|
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("&", " и "),
|
|||
|
|
("@", " собака "),
|
|||
|
|
("%", " процентов "),
|
|||
|
|
("#", " номер "),
|
|||
|
|
("$", " доллар "),
|
|||
|
|
("£", " фунт "),
|
|||
|
|
("°", " градус "),
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"nl": [
|
|||
|
|
# Dutch
|
|||
|
|
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("&", " en "),
|
|||
|
|
("@", " bij "),
|
|||
|
|
("%", " procent "),
|
|||
|
|
("#", " hekje "),
|
|||
|
|
("$", " dollar "),
|
|||
|
|
("£", " pond "),
|
|||
|
|
("°", " graden "),
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"tr": [
|
|||
|
|
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("&", " ve "),
|
|||
|
|
("@", " at "),
|
|||
|
|
("%", " yüzde "),
|
|||
|
|
("#", " diyez "),
|
|||
|
|
("$", " dolar "),
|
|||
|
|
("£", " sterlin "),
|
|||
|
|
("°", " derece "),
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"hu": [
|
|||
|
|
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("&", " és "),
|
|||
|
|
("@", " kukac "),
|
|||
|
|
("%", " százalék "),
|
|||
|
|
("#", " kettőskereszt "),
|
|||
|
|
("$", " dollár "),
|
|||
|
|
("£", " font "),
|
|||
|
|
("°", " fok "),
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
"ko": [
|
|||
|
|
# Korean
|
|||
|
|
(re.compile(r"%s" % re.escape(x[0]), re.IGNORECASE), x[1])
|
|||
|
|
for x in [
|
|||
|
|
("&", " 그리고 "),
|
|||
|
|
("@", " 에 "),
|
|||
|
|
("%", " 퍼센트 "),
|
|||
|
|
("#", " 번호 "),
|
|||
|
|
("$", " 달러 "),
|
|||
|
|
("£", " 파운드 "),
|
|||
|
|
("°", " 도 "),
|
|||
|
|
]
|
|||
|
|
],
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def expand_symbols_multilingual(text, lang="en"):
|
|||
|
|
for regex, replacement in _symbols_multilingual[lang]:
|
|||
|
|
text = re.sub(regex, replacement, text)
|
|||
|
|
text = text.replace(" ", " ") # Ensure there are no double spaces
|
|||
|
|
return text.strip()
|
|||
|
|
|
|||
|
|
|
|||
|
|
_ordinal_re = {
|
|||
|
|
"en": re.compile(r"([0-9]+)(st|nd|rd|th)"),
|
|||
|
|
"es": re.compile(r"([0-9]+)(º|ª|er|o|a|os|as)"),
|
|||
|
|
"fr": re.compile(r"([0-9]+)(º|ª|er|re|e|ème)"),
|
|||
|
|
"de": re.compile(r"([0-9]+)(st|nd|rd|th|º|ª|\.(?=\s|$))"),
|
|||
|
|
"pt": re.compile(r"([0-9]+)(º|ª|o|a|os|as)"),
|
|||
|
|
"it": re.compile(r"([0-9]+)(º|°|ª|o|a|i|e)"),
|
|||
|
|
"pl": re.compile(r"([0-9]+)(º|ª|st|nd|rd|th)"),
|
|||
|
|
"ar": re.compile(r"([0-9]+)(ون|ين|ث|ر|ى)"),
|
|||
|
|
"cs": re.compile(r"([0-9]+)\.(?=\s|$)"), # In Czech, a dot is often used after the number to indicate ordinals.
|
|||
|
|
"ru": re.compile(r"([0-9]+)(-й|-я|-е|-ое|-ье|-го)"),
|
|||
|
|
"nl": re.compile(r"([0-9]+)(de|ste|e)"),
|
|||
|
|
"tr": re.compile(r"([0-9]+)(\.|inci|nci|uncu|üncü|\.)"),
|
|||
|
|
"hu": re.compile(r"([0-9]+)(\.|adik|edik|odik|edik|ödik|ödike|ik)"),
|
|||
|
|
"ko": re.compile(r"([0-9]+)(번째|번|차|째)"),
|
|||
|
|
}
|
|||
|
|
_number_re = re.compile(r"[0-9]+")
|
|||
|
|
_currency_re = {
|
|||
|
|
"USD": re.compile(r"((\$[0-9\.\,]*[0-9]+)|([0-9\.\,]*[0-9]+\$))"),
|
|||
|
|
"GBP": re.compile(r"((£[0-9\.\,]*[0-9]+)|([0-9\.\,]*[0-9]+£))"),
|
|||
|
|
"EUR": re.compile(r"(([0-9\.\,]*[0-9]+€)|((€[0-9\.\,]*[0-9]+)))"),
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
_comma_number_re = re.compile(r"\b\d{1,3}(,\d{3})*(\.\d+)?\b")
|
|||
|
|
_dot_number_re = re.compile(r"\b\d{1,3}(.\d{3})*(\,\d+)?\b")
|
|||
|
|
_decimal_number_re = re.compile(r"([0-9]+[.,][0-9]+)")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _remove_commas(m):
|
|||
|
|
text = m.group(0)
|
|||
|
|
if "," in text:
|
|||
|
|
text = text.replace(",", "")
|
|||
|
|
return text
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _remove_dots(m):
|
|||
|
|
text = m.group(0)
|
|||
|
|
if "." in text:
|
|||
|
|
text = text.replace(".", "")
|
|||
|
|
return text
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _expand_decimal_point(m, lang="en"):
|
|||
|
|
amount = m.group(1).replace(",", ".")
|
|||
|
|
return num2words(float(amount), lang=lang if lang != "cs" else "cz")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _expand_currency(m, lang="en", currency="USD"):
|
|||
|
|
amount = float((re.sub(r"[^\d.]", "", m.group(0).replace(",", "."))))
|
|||
|
|
full_amount = num2words(amount, to="currency", currency=currency, lang=lang if lang != "cs" else "cz")
|
|||
|
|
|
|||
|
|
and_equivalents = {
|
|||
|
|
"en": ", ",
|
|||
|
|
"es": " con ",
|
|||
|
|
"fr": " et ",
|
|||
|
|
"de": " und ",
|
|||
|
|
"pt": " e ",
|
|||
|
|
"it": " e ",
|
|||
|
|
"pl": ", ",
|
|||
|
|
"cs": ", ",
|
|||
|
|
"ru": ", ",
|
|||
|
|
"nl": ", ",
|
|||
|
|
"ar": ", ",
|
|||
|
|
"tr": ", ",
|
|||
|
|
"hu": ", ",
|
|||
|
|
"ko": ", ",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if amount.is_integer():
|
|||
|
|
last_and = full_amount.rfind(and_equivalents[lang])
|
|||
|
|
if last_and != -1:
|
|||
|
|
full_amount = full_amount[:last_and]
|
|||
|
|
|
|||
|
|
return full_amount
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _expand_ordinal(m, lang="en"):
|
|||
|
|
return num2words(int(m.group(1)), ordinal=True, lang=lang if lang != "cs" else "cz")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _expand_number(m, lang="en"):
|
|||
|
|
return num2words(int(m.group(0)), lang=lang if lang != "cs" else "cz")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def expand_numbers_multilingual(text, lang="en"):
|
|||
|
|
if lang == "zh":
|
|||
|
|
text = zh_num2words()(text)
|
|||
|
|
else:
|
|||
|
|
if lang in ["en", "ru"]:
|
|||
|
|
text = re.sub(_comma_number_re, _remove_commas, text)
|
|||
|
|
else:
|
|||
|
|
text = re.sub(_dot_number_re, _remove_dots, text)
|
|||
|
|
try:
|
|||
|
|
text = re.sub(_currency_re["GBP"], lambda m: _expand_currency(m, lang, "GBP"), text)
|
|||
|
|
text = re.sub(_currency_re["USD"], lambda m: _expand_currency(m, lang, "USD"), text)
|
|||
|
|
text = re.sub(_currency_re["EUR"], lambda m: _expand_currency(m, lang, "EUR"), text)
|
|||
|
|
except:
|
|||
|
|
pass
|
|||
|
|
if lang != "tr":
|
|||
|
|
text = re.sub(_decimal_number_re, lambda m: _expand_decimal_point(m, lang), text)
|
|||
|
|
text = re.sub(_ordinal_re[lang], lambda m: _expand_ordinal(m, lang), text)
|
|||
|
|
text = re.sub(_number_re, lambda m: _expand_number(m, lang), text)
|
|||
|
|
return text
|
|||
|
|
|
|||
|
|
|
|||
|
|
def lowercase(text):
|
|||
|
|
return text.lower()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def collapse_whitespace(text):
|
|||
|
|
return re.sub(_whitespace_re, " ", text)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def multilingual_cleaners(text, lang):
|
|||
|
|
text = text.replace('"', "")
|
|||
|
|
if lang == "tr":
|
|||
|
|
text = text.replace("İ", "i")
|
|||
|
|
text = text.replace("Ö", "ö")
|
|||
|
|
text = text.replace("Ü", "ü")
|
|||
|
|
text = lowercase(text)
|
|||
|
|
text = expand_numbers_multilingual(text, lang)
|
|||
|
|
text = expand_abbreviations_multilingual(text, lang)
|
|||
|
|
text = expand_symbols_multilingual(text, lang=lang)
|
|||
|
|
text = collapse_whitespace(text)
|
|||
|
|
return text
|
|||
|
|
|
|||
|
|
|
|||
|
|
def basic_cleaners(text):
|
|||
|
|
"""Basic pipeline that lowercases and collapses whitespace without transliteration."""
|
|||
|
|
text = lowercase(text)
|
|||
|
|
text = collapse_whitespace(text)
|
|||
|
|
return text
|
|||
|
|
|
|||
|
|
|
|||
|
|
def chinese_transliterate(text):
|
|||
|
|
return "".join(
|
|||
|
|
[p[0] for p in pypinyin.pinyin(text, style=pypinyin.Style.TONE3, heteronym=False, neutral_tone_with_five=True)]
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def japanese_cleaners(text, katsu):
|
|||
|
|
text = katsu.romaji(text)
|
|||
|
|
text = lowercase(text)
|
|||
|
|
return text
|
|||
|
|
|
|||
|
|
|
|||
|
|
def korean_transliterate(text):
|
|||
|
|
r = Transliter(academic)
|
|||
|
|
return r.translit(text)
|
|||
|
|
|
|||
|
|
|
|||
|
|
DEFAULT_VOCAB_FILE = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../data/tokenizer.json")
|
|||
|
|
|
|||
|
|
|
|||
|
|
class VoiceBpeTokenizer:
|
|||
|
|
def __init__(self, vocab_file=None):
|
|||
|
|
self.tokenizer = None
|
|||
|
|
if vocab_file is not None:
|
|||
|
|
self.tokenizer = Tokenizer.from_file(vocab_file)
|
|||
|
|
self.char_limits = {
|
|||
|
|
"en": 250,
|
|||
|
|
"de": 253,
|
|||
|
|
"fr": 273,
|
|||
|
|
"es": 239,
|
|||
|
|
"it": 213,
|
|||
|
|
"pt": 203,
|
|||
|
|
"pl": 224,
|
|||
|
|
"zh": 82,
|
|||
|
|
"ar": 166,
|
|||
|
|
"cs": 186,
|
|||
|
|
"ru": 182,
|
|||
|
|
"nl": 251,
|
|||
|
|
"tr": 226,
|
|||
|
|
"ja": 71,
|
|||
|
|
"hu": 224,
|
|||
|
|
"ko": 95,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@cached_property
|
|||
|
|
def katsu(self):
|
|||
|
|
import cutlet
|
|||
|
|
|
|||
|
|
return cutlet.Cutlet()
|
|||
|
|
|
|||
|
|
def check_input_length(self, txt, lang):
|
|||
|
|
lang = lang.split("-")[0] # remove the region
|
|||
|
|
limit = self.char_limits.get(lang, 250)
|
|||
|
|
if len(txt) > limit:
|
|||
|
|
print(
|
|||
|
|
f"[!] Warning: The text length exceeds the character limit of {limit} for language '{lang}', this might cause truncated audio."
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def preprocess_text(self, txt, lang):
|
|||
|
|
if lang in {"ar", "cs", "de", "en", "es", "fr", "hu", "it", "nl", "pl", "pt", "ru", "tr", "zh", "ko"}:
|
|||
|
|
txt = multilingual_cleaners(txt, lang)
|
|||
|
|
if lang == "zh":
|
|||
|
|
txt = chinese_transliterate(txt)
|
|||
|
|
if lang == "ko":
|
|||
|
|
txt = korean_transliterate(txt)
|
|||
|
|
elif lang == "ja":
|
|||
|
|
txt = japanese_cleaners(txt, self.katsu)
|
|||
|
|
elif lang == "hi":
|
|||
|
|
# @manmay will implement this
|
|||
|
|
txt = basic_cleaners(txt)
|
|||
|
|
else:
|
|||
|
|
raise NotImplementedError(f"Language '{lang}' is not supported.")
|
|||
|
|
return txt
|
|||
|
|
|
|||
|
|
def encode(self, txt, lang):
|
|||
|
|
lang = lang.split("-")[0] # remove the region
|
|||
|
|
self.check_input_length(txt, lang)
|
|||
|
|
txt = self.preprocess_text(txt, lang)
|
|||
|
|
lang = "zh-cn" if lang == "zh" else lang
|
|||
|
|
txt = f"[{lang}]{txt}"
|
|||
|
|
txt = txt.replace(" ", "[SPACE]")
|
|||
|
|
return self.tokenizer.encode(txt).ids
|
|||
|
|
|
|||
|
|
def decode(self, seq):
|
|||
|
|
if isinstance(seq, torch.Tensor):
|
|||
|
|
seq = seq.cpu().numpy()
|
|||
|
|
txt = self.tokenizer.decode(seq, skip_special_tokens=False).replace(" ", "")
|
|||
|
|
txt = txt.replace("[SPACE]", " ")
|
|||
|
|
txt = txt.replace("[STOP]", "")
|
|||
|
|
txt = txt.replace("[UNK]", "")
|
|||
|
|
return txt
|
|||
|
|
|
|||
|
|
def __len__(self):
|
|||
|
|
return self.tokenizer.get_vocab_size()
|
|||
|
|
|
|||
|
|
def get_number_tokens(self):
|
|||
|
|
return max(self.tokenizer.get_vocab().values()) + 1
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_expand_numbers_multilingual():
|
|||
|
|
test_cases = [
|
|||
|
|
# English
|
|||
|
|
("In 12.5 seconds.", "In twelve point five seconds.", "en"),
|
|||
|
|
("There were 50 soldiers.", "There were fifty soldiers.", "en"),
|
|||
|
|
("This is a 1st test", "This is a first test", "en"),
|
|||
|
|
("That will be $20 sir.", "That will be twenty dollars sir.", "en"),
|
|||
|
|
("That will be 20€ sir.", "That will be twenty euro sir.", "en"),
|
|||
|
|
("That will be 20.15€ sir.", "That will be twenty euro, fifteen cents sir.", "en"),
|
|||
|
|
("That's 100,000.5.", "That's one hundred thousand point five.", "en"),
|
|||
|
|
# French
|
|||
|
|
("En 12,5 secondes.", "En douze virgule cinq secondes.", "fr"),
|
|||
|
|
("Il y avait 50 soldats.", "Il y avait cinquante soldats.", "fr"),
|
|||
|
|
("Ceci est un 1er test", "Ceci est un premier test", "fr"),
|
|||
|
|
("Cela vous fera $20 monsieur.", "Cela vous fera vingt dollars monsieur.", "fr"),
|
|||
|
|
("Cela vous fera 20€ monsieur.", "Cela vous fera vingt euros monsieur.", "fr"),
|
|||
|
|
("Cela vous fera 20,15€ monsieur.", "Cela vous fera vingt euros et quinze centimes monsieur.", "fr"),
|
|||
|
|
("Ce sera 100.000,5.", "Ce sera cent mille virgule cinq.", "fr"),
|
|||
|
|
# German
|
|||
|
|
("In 12,5 Sekunden.", "In zwölf Komma fünf Sekunden.", "de"),
|
|||
|
|
("Es gab 50 Soldaten.", "Es gab fünfzig Soldaten.", "de"),
|
|||
|
|
("Dies ist ein 1. Test", "Dies ist ein erste Test", "de"), # Issue with gender
|
|||
|
|
("Das macht $20 Herr.", "Das macht zwanzig Dollar Herr.", "de"),
|
|||
|
|
("Das macht 20€ Herr.", "Das macht zwanzig Euro Herr.", "de"),
|
|||
|
|
("Das macht 20,15€ Herr.", "Das macht zwanzig Euro und fünfzehn Cent Herr.", "de"),
|
|||
|
|
# Spanish
|
|||
|
|
("En 12,5 segundos.", "En doce punto cinco segundos.", "es"),
|
|||
|
|
("Había 50 soldados.", "Había cincuenta soldados.", "es"),
|
|||
|
|
("Este es un 1er test", "Este es un primero test", "es"),
|
|||
|
|
("Eso le costará $20 señor.", "Eso le costará veinte dólares señor.", "es"),
|
|||
|
|
("Eso le costará 20€ señor.", "Eso le costará veinte euros señor.", "es"),
|
|||
|
|
("Eso le costará 20,15€ señor.", "Eso le costará veinte euros con quince céntimos señor.", "es"),
|
|||
|
|
# Italian
|
|||
|
|
("In 12,5 secondi.", "In dodici virgola cinque secondi.", "it"),
|
|||
|
|
("C'erano 50 soldati.", "C'erano cinquanta soldati.", "it"),
|
|||
|
|
("Questo è un 1° test", "Questo è un primo test", "it"),
|
|||
|
|
("Ti costerà $20 signore.", "Ti costerà venti dollari signore.", "it"),
|
|||
|
|
("Ti costerà 20€ signore.", "Ti costerà venti euro signore.", "it"),
|
|||
|
|
("Ti costerà 20,15€ signore.", "Ti costerà venti euro e quindici centesimi signore.", "it"),
|
|||
|
|
# Portuguese
|
|||
|
|
("Em 12,5 segundos.", "Em doze vírgula cinco segundos.", "pt"),
|
|||
|
|
("Havia 50 soldados.", "Havia cinquenta soldados.", "pt"),
|
|||
|
|
("Este é um 1º teste", "Este é um primeiro teste", "pt"),
|
|||
|
|
("Isso custará $20 senhor.", "Isso custará vinte dólares senhor.", "pt"),
|
|||
|
|
("Isso custará 20€ senhor.", "Isso custará vinte euros senhor.", "pt"),
|
|||
|
|
(
|
|||
|
|
"Isso custará 20,15€ senhor.",
|
|||
|
|
"Isso custará vinte euros e quinze cêntimos senhor.",
|
|||
|
|
"pt",
|
|||
|
|
), # "cêntimos" should be "centavos" num2words issue
|
|||
|
|
# Polish
|
|||
|
|
("W 12,5 sekundy.", "W dwanaście przecinek pięć sekundy.", "pl"),
|
|||
|
|
("Było 50 żołnierzy.", "Było pięćdziesiąt żołnierzy.", "pl"),
|
|||
|
|
("To będzie kosztować 20€ panie.", "To będzie kosztować dwadzieścia euro panie.", "pl"),
|
|||
|
|
("To będzie kosztować 20,15€ panie.", "To będzie kosztować dwadzieścia euro, piętnaście centów panie.", "pl"),
|
|||
|
|
# Arabic
|
|||
|
|
("في الـ 12,5 ثانية.", "في الـ اثنا عشر , خمسون ثانية.", "ar"),
|
|||
|
|
("كان هناك 50 جنديًا.", "كان هناك خمسون جنديًا.", "ar"),
|
|||
|
|
# ("ستكون النتيجة $20 يا سيد.", 'ستكون النتيجة عشرون دولار يا سيد.', 'ar'), # $ and € are mising from num2words
|
|||
|
|
# ("ستكون النتيجة 20€ يا سيد.", 'ستكون النتيجة عشرون يورو يا سيد.', 'ar'),
|
|||
|
|
# Czech
|
|||
|
|
("Za 12,5 vteřiny.", "Za dvanáct celá pět vteřiny.", "cs"),
|
|||
|
|
("Bylo tam 50 vojáků.", "Bylo tam padesát vojáků.", "cs"),
|
|||
|
|
("To bude stát 20€ pane.", "To bude stát dvacet euro pane.", "cs"),
|
|||
|
|
("To bude 20.15€ pane.", "To bude dvacet euro, patnáct centů pane.", "cs"),
|
|||
|
|
# Russian
|
|||
|
|
("Через 12.5 секунды.", "Через двенадцать запятая пять секунды.", "ru"),
|
|||
|
|
("Там было 50 солдат.", "Там было пятьдесят солдат.", "ru"),
|
|||
|
|
("Это будет 20.15€ сэр.", "Это будет двадцать евро, пятнадцать центов сэр.", "ru"),
|
|||
|
|
("Это будет стоить 20€ господин.", "Это будет стоить двадцать евро господин.", "ru"),
|
|||
|
|
# Dutch
|
|||
|
|
("In 12,5 seconden.", "In twaalf komma vijf seconden.", "nl"),
|
|||
|
|
("Er waren 50 soldaten.", "Er waren vijftig soldaten.", "nl"),
|
|||
|
|
("Dat wordt dan $20 meneer.", "Dat wordt dan twintig dollar meneer.", "nl"),
|
|||
|
|
("Dat wordt dan 20€ meneer.", "Dat wordt dan twintig euro meneer.", "nl"),
|
|||
|
|
# Chinese (Simplified)
|
|||
|
|
("在12.5秒内", "在十二点五秒内", "zh"),
|
|||
|
|
("有50名士兵", "有五十名士兵", "zh"),
|
|||
|
|
# ("那将是$20先生", '那将是二十美元先生', 'zh'), currency doesn't work
|
|||
|
|
# ("那将是20€先生", '那将是二十欧元先生', 'zh'),
|
|||
|
|
# Turkish
|
|||
|
|
# ("12,5 saniye içinde.", 'On iki virgül beş saniye içinde.', 'tr'), # decimal doesn't work for TR
|
|||
|
|
("50 asker vardı.", "elli asker vardı.", "tr"),
|
|||
|
|
("Bu 1. test", "Bu birinci test", "tr"),
|
|||
|
|
# ("Bu 100.000,5.", 'Bu yüz bin virgül beş.', 'tr'),
|
|||
|
|
# Hungarian
|
|||
|
|
("12,5 másodperc alatt.", "tizenkettő egész öt tized másodperc alatt.", "hu"),
|
|||
|
|
("50 katona volt.", "ötven katona volt.", "hu"),
|
|||
|
|
("Ez az 1. teszt", "Ez az első teszt", "hu"),
|
|||
|
|
# Korean
|
|||
|
|
("12.5 초 안에.", "십이 점 다섯 초 안에.", "ko"),
|
|||
|
|
("50 명의 병사가 있었다.", "오십 명의 병사가 있었다.", "ko"),
|
|||
|
|
("이것은 1 번째 테스트입니다", "이것은 첫 번째 테스트입니다", "ko"),
|
|||
|
|
]
|
|||
|
|
for a, b, lang in test_cases:
|
|||
|
|
out = expand_numbers_multilingual(a, lang=lang)
|
|||
|
|
assert out == b, f"'{out}' vs '{b}'"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_abbreviations_multilingual():
|
|||
|
|
test_cases = [
|
|||
|
|
# English
|
|||
|
|
("Hello Mr. Smith.", "Hello mister Smith.", "en"),
|
|||
|
|
("Dr. Jones is here.", "doctor Jones is here.", "en"),
|
|||
|
|
# Spanish
|
|||
|
|
("Hola Sr. Garcia.", "Hola señor Garcia.", "es"),
|
|||
|
|
("La Dra. Martinez es muy buena.", "La doctora Martinez es muy buena.", "es"),
|
|||
|
|
# French
|
|||
|
|
("Bonjour Mr. Dupond.", "Bonjour monsieur Dupond.", "fr"),
|
|||
|
|
("Mme. Moreau est absente aujourd'hui.", "madame Moreau est absente aujourd'hui.", "fr"),
|
|||
|
|
# German
|
|||
|
|
("Frau Dr. Müller ist sehr klug.", "Frau doktor Müller ist sehr klug.", "de"),
|
|||
|
|
# Portuguese
|
|||
|
|
("Olá Sr. Silva.", "Olá senhor Silva.", "pt"),
|
|||
|
|
("Dra. Costa, você está disponível?", "doutora Costa, você está disponível?", "pt"),
|
|||
|
|
# Italian
|
|||
|
|
("Buongiorno, Sig. Rossi.", "Buongiorno, signore Rossi.", "it"),
|
|||
|
|
# ("Sig.ra Bianchi, posso aiutarti?", 'signora Bianchi, posso aiutarti?', 'it'), # Issue with matching that pattern
|
|||
|
|
# Polish
|
|||
|
|
("Dzień dobry, P. Kowalski.", "Dzień dobry, pani Kowalski.", "pl"),
|
|||
|
|
("M. Nowak, czy mogę zadać pytanie?", "pan Nowak, czy mogę zadać pytanie?", "pl"),
|
|||
|
|
# Czech
|
|||
|
|
("P. Novák", "pan Novák", "cs"),
|
|||
|
|
("Dr. Vojtěch", "doktor Vojtěch", "cs"),
|
|||
|
|
# Dutch
|
|||
|
|
("Dhr. Jansen", "de heer Jansen", "nl"),
|
|||
|
|
("Mevr. de Vries", "mevrouw de Vries", "nl"),
|
|||
|
|
# Russian
|
|||
|
|
("Здравствуйте Г-н Иванов.", "Здравствуйте господин Иванов.", "ru"),
|
|||
|
|
("Д-р Смирнов здесь, чтобы увидеть вас.", "доктор Смирнов здесь, чтобы увидеть вас.", "ru"),
|
|||
|
|
# Turkish
|
|||
|
|
("Merhaba B. Yılmaz.", "Merhaba bay Yılmaz.", "tr"),
|
|||
|
|
("Dr. Ayşe burada.", "doktor Ayşe burada.", "tr"),
|
|||
|
|
# Hungarian
|
|||
|
|
("Dr. Szabó itt van.", "doktor Szabó itt van.", "hu"),
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
for a, b, lang in test_cases:
|
|||
|
|
out = expand_abbreviations_multilingual(a, lang=lang)
|
|||
|
|
assert out == b, f"'{out}' vs '{b}'"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_symbols_multilingual():
|
|||
|
|
test_cases = [
|
|||
|
|
("I have 14% battery", "I have 14 percent battery", "en"),
|
|||
|
|
("Te veo @ la fiesta", "Te veo arroba la fiesta", "es"),
|
|||
|
|
("J'ai 14° de fièvre", "J'ai 14 degrés de fièvre", "fr"),
|
|||
|
|
("Die Rechnung beträgt £ 20", "Die Rechnung beträgt pfund 20", "de"),
|
|||
|
|
("O meu email é ana&joao@gmail.com", "O meu email é ana e joao arroba gmail.com", "pt"),
|
|||
|
|
("linguaggio di programmazione C#", "linguaggio di programmazione C cancelletto", "it"),
|
|||
|
|
("Moja temperatura to 36.6°", "Moja temperatura to 36.6 stopnie", "pl"),
|
|||
|
|
("Mám 14% baterie", "Mám 14 procento baterie", "cs"),
|
|||
|
|
("Těším se na tebe @ party", "Těším se na tebe na party", "cs"),
|
|||
|
|
("У меня 14% заряда", "У меня 14 процентов заряда", "ru"),
|
|||
|
|
("Я буду @ дома", "Я буду собака дома", "ru"),
|
|||
|
|
("Ik heb 14% batterij", "Ik heb 14 procent batterij", "nl"),
|
|||
|
|
("Ik zie je @ het feest", "Ik zie je bij het feest", "nl"),
|
|||
|
|
("لدي 14% في البطارية", "لدي 14 في المئة في البطارية", "ar"),
|
|||
|
|
("我的电量为 14%", "我的电量为 14 百分之", "zh"),
|
|||
|
|
("Pilim %14 dolu.", "Pilim yüzde 14 dolu.", "tr"),
|
|||
|
|
("Az akkumulátorom töltöttsége 14%", "Az akkumulátorom töltöttsége 14 százalék", "hu"),
|
|||
|
|
("배터리 잔량이 14%입니다.", "배터리 잔량이 14 퍼센트입니다.", "ko"),
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
for a, b, lang in test_cases:
|
|||
|
|
out = expand_symbols_multilingual(a, lang=lang)
|
|||
|
|
assert out == b, f"'{out}' vs '{b}'"
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
test_expand_numbers_multilingual()
|
|||
|
|
test_abbreviations_multilingual()
|
|||
|
|
test_symbols_multilingual()
|