33 lines
1.3 KiB
Python
33 lines
1.3 KiB
Python
from pathlib import Path
|
|
import urllib.request
|
|
import os
|
|
import logging
|
|
|
|
from .exceptions import FastTextModelNotFoundException
|
|
from .utils import dateparser_model_home, create_data_model_home
|
|
|
|
|
|
def fasttext_downloader(model_name):
|
|
create_data_model_home()
|
|
models = {
|
|
"small": "https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.ftz",
|
|
"large": "https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin"
|
|
}
|
|
if model_name not in models:
|
|
message = "dateparser-download: Couldn't find a model called \"{}\". Supported models are: {}".format(
|
|
model_name, ", ".join(models.keys())
|
|
)
|
|
raise FastTextModelNotFoundException(message)
|
|
|
|
models_directory_path = os.path.join(dateparser_model_home, (model_name + ".bin"))
|
|
|
|
if not Path(models_directory_path).is_file():
|
|
model_url = models[model_name]
|
|
logging.info("dateparser-download: Downloading model \"{}\" from \"{}\"...".format(model_name, model_url))
|
|
try:
|
|
urllib.request.urlretrieve(model_url, models_directory_path)
|
|
except urllib.error.HTTPError as e:
|
|
raise Exception("dateparser-download: Fasttext model cannot be downloaded due to HTTP error") from e
|
|
else:
|
|
logging.info("dateparser-download: The model \"{}\" is already downloaded".format(model_name))
|