299 lines
13 KiB
Python
299 lines
13 KiB
Python
|
# coding=utf-8
|
||
|
|
# Copyright 2023-present, the HuggingFace Inc. team.
|
||
|
|
#
|
||
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||
|
|
# you may not use this file except in compliance with the License.
|
||
|
|
# You may obtain a copy of the License at
|
||
|
|
#
|
||
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||
|
|
#
|
||
|
|
# Unless required by applicable law or agreed to in writing, software
|
||
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
|
|
# See the License for the specific language governing permissions and
|
||
|
|
# limitations under the License.
|
||
|
|
"""Contains command to upload a repo or file with the CLI.
|
||
|
|
|
||
|
|
Usage:
|
||
|
|
# Upload file (implicit)
|
||
|
|
huggingface-cli upload my-cool-model ./my-cool-model.safetensors
|
||
|
|
|
||
|
|
# Upload file (explicit)
|
||
|
|
huggingface-cli upload my-cool-model ./my-cool-model.safetensors model.safetensors
|
||
|
|
|
||
|
|
# Upload directory (implicit). If `my-cool-model/` is a directory it will be uploaded, otherwise an exception is raised.
|
||
|
|
huggingface-cli upload my-cool-model
|
||
|
|
|
||
|
|
# Upload directory (explicit)
|
||
|
|
huggingface-cli upload my-cool-model ./models/my-cool-model .
|
||
|
|
|
||
|
|
# Upload filtered directory (example: tensorboard logs except for the last run)
|
||
|
|
huggingface-cli upload my-cool-model ./model/training /logs --include "*.tfevents.*" --exclude "*20230905*"
|
||
|
|
|
||
|
|
# Upload private dataset
|
||
|
|
huggingface-cli upload Wauplin/my-cool-dataset ./data . --repo-type=dataset --private
|
||
|
|
|
||
|
|
# Upload with token
|
||
|
|
huggingface-cli upload Wauplin/my-cool-model --token=hf_****
|
||
|
|
|
||
|
|
# Sync local Space with Hub (upload new files, delete removed files)
|
||
|
|
huggingface-cli upload Wauplin/space-example --repo-type=space --exclude="/logs/*" --delete="*" --commit-message="Sync local Space with Hub"
|
||
|
|
|
||
|
|
# Schedule commits every 30 minutes
|
||
|
|
huggingface-cli upload Wauplin/my-cool-model --every=30
|
||
|
|
"""
|
||
|
|
|
||
|
|
import os
|
||
|
|
import time
|
||
|
|
import warnings
|
||
|
|
from argparse import Namespace, _SubParsersAction
|
||
|
|
from typing import List, Optional
|
||
|
|
|
||
|
|
from huggingface_hub import logging
|
||
|
|
from huggingface_hub._commit_scheduler import CommitScheduler
|
||
|
|
from huggingface_hub.commands import BaseHuggingfaceCLICommand
|
||
|
|
from huggingface_hub.constants import HF_HUB_ENABLE_HF_TRANSFER
|
||
|
|
from huggingface_hub.hf_api import HfApi
|
||
|
|
from huggingface_hub.utils import RevisionNotFoundError, disable_progress_bars, enable_progress_bars
|
||
|
|
|
||
|
|
|
||
|
|
logger = logging.get_logger(__name__)
|
||
|
|
|
||
|
|
|
||
|
|
class UploadCommand(BaseHuggingfaceCLICommand):
|
||
|
|
@staticmethod
|
||
|
|
def register_subcommand(parser: _SubParsersAction):
|
||
|
|
upload_parser = parser.add_parser("upload", help="Upload a file or a folder to a repo on the Hub")
|
||
|
|
upload_parser.add_argument(
|
||
|
|
"repo_id", type=str, help="The ID of the repo to upload to (e.g. `username/repo-name`)."
|
||
|
|
)
|
||
|
|
upload_parser.add_argument(
|
||
|
|
"local_path", nargs="?", help="Local path to the file or folder to upload. Defaults to current directory."
|
||
|
|
)
|
||
|
|
upload_parser.add_argument(
|
||
|
|
"path_in_repo",
|
||
|
|
nargs="?",
|
||
|
|
help="Path of the file or folder in the repo. Defaults to the relative path of the file or folder.",
|
||
|
|
)
|
||
|
|
upload_parser.add_argument(
|
||
|
|
"--repo-type",
|
||
|
|
choices=["model", "dataset", "space"],
|
||
|
|
default="model",
|
||
|
|
help="Type of the repo to upload to (e.g. `dataset`).",
|
||
|
|
)
|
||
|
|
upload_parser.add_argument(
|
||
|
|
"--revision",
|
||
|
|
type=str,
|
||
|
|
help=(
|
||
|
|
"An optional Git revision to push to. It can be a branch name or a PR reference. If revision does not"
|
||
|
|
" exist and `--create-pr` is not set, a branch will be automatically created."
|
||
|
|
),
|
||
|
|
)
|
||
|
|
upload_parser.add_argument(
|
||
|
|
"--private",
|
||
|
|
action="store_true",
|
||
|
|
help=(
|
||
|
|
"Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already"
|
||
|
|
" exists."
|
||
|
|
),
|
||
|
|
)
|
||
|
|
upload_parser.add_argument("--include", nargs="*", type=str, help="Glob patterns to match files to upload.")
|
||
|
|
upload_parser.add_argument(
|
||
|
|
"--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to upload."
|
||
|
|
)
|
||
|
|
upload_parser.add_argument(
|
||
|
|
"--delete",
|
||
|
|
nargs="*",
|
||
|
|
type=str,
|
||
|
|
help="Glob patterns for file to be deleted from the repo while committing.",
|
||
|
|
)
|
||
|
|
upload_parser.add_argument(
|
||
|
|
"--commit-message", type=str, help="The summary / title / first line of the generated commit."
|
||
|
|
)
|
||
|
|
upload_parser.add_argument("--commit-description", type=str, help="The description of the generated commit.")
|
||
|
|
upload_parser.add_argument(
|
||
|
|
"--create-pr", action="store_true", help="Whether to upload content as a new Pull Request."
|
||
|
|
)
|
||
|
|
upload_parser.add_argument(
|
||
|
|
"--every",
|
||
|
|
type=float,
|
||
|
|
help="If set, a background job is scheduled to create commits every `every` minutes.",
|
||
|
|
)
|
||
|
|
upload_parser.add_argument(
|
||
|
|
"--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens"
|
||
|
|
)
|
||
|
|
upload_parser.add_argument(
|
||
|
|
"--quiet",
|
||
|
|
action="store_true",
|
||
|
|
help="If True, progress bars are disabled and only the path to the uploaded files is printed.",
|
||
|
|
)
|
||
|
|
upload_parser.set_defaults(func=UploadCommand)
|
||
|
|
|
||
|
|
def __init__(self, args: Namespace) -> None:
|
||
|
|
self.repo_id: str = args.repo_id
|
||
|
|
self.repo_type: Optional[str] = args.repo_type
|
||
|
|
self.revision: Optional[str] = args.revision
|
||
|
|
self.private: bool = args.private
|
||
|
|
|
||
|
|
self.include: Optional[List[str]] = args.include
|
||
|
|
self.exclude: Optional[List[str]] = args.exclude
|
||
|
|
self.delete: Optional[List[str]] = args.delete
|
||
|
|
|
||
|
|
self.commit_message: Optional[str] = args.commit_message
|
||
|
|
self.commit_description: Optional[str] = args.commit_description
|
||
|
|
self.create_pr: bool = args.create_pr
|
||
|
|
self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli")
|
||
|
|
self.quiet: bool = args.quiet # disable warnings and progress bars
|
||
|
|
|
||
|
|
# Check `--every` is valid
|
||
|
|
if args.every is not None and args.every <= 0:
|
||
|
|
raise ValueError(f"`every` must be a positive value (got '{args.every}')")
|
||
|
|
self.every: Optional[float] = args.every
|
||
|
|
|
||
|
|
# Resolve `local_path` and `path_in_repo`
|
||
|
|
repo_name: str = args.repo_id.split("/")[-1] # e.g. "Wauplin/my-cool-model" => "my-cool-model"
|
||
|
|
self.local_path: str
|
||
|
|
self.path_in_repo: str
|
||
|
|
if args.local_path is None and os.path.isfile(repo_name):
|
||
|
|
# Implicit case 1: user provided only a repo_id which happen to be a local file as well => upload it with same name
|
||
|
|
self.local_path = repo_name
|
||
|
|
self.path_in_repo = repo_name
|
||
|
|
elif args.local_path is None and os.path.isdir(repo_name):
|
||
|
|
# Implicit case 2: user provided only a repo_id which happen to be a local folder as well => upload it at root
|
||
|
|
self.local_path = repo_name
|
||
|
|
self.path_in_repo = "."
|
||
|
|
elif args.local_path is None:
|
||
|
|
# Implicit case 3: user provided only a repo_id that does not match a local file or folder
|
||
|
|
# => the user must explicitly provide a local_path => raise exception
|
||
|
|
raise ValueError(f"'{repo_name}' is not a local file or folder. Please set `local_path` explicitly.")
|
||
|
|
elif args.path_in_repo is None and os.path.isfile(args.local_path):
|
||
|
|
# Explicit local path to file, no path in repo => upload it at root with same name
|
||
|
|
self.local_path = args.local_path
|
||
|
|
self.path_in_repo = os.path.basename(args.local_path)
|
||
|
|
elif args.path_in_repo is None:
|
||
|
|
# Explicit local path to folder, no path in repo => upload at root
|
||
|
|
self.local_path = args.local_path
|
||
|
|
self.path_in_repo = "."
|
||
|
|
else:
|
||
|
|
# Finally, if both paths are explicit
|
||
|
|
self.local_path = args.local_path
|
||
|
|
self.path_in_repo = args.path_in_repo
|
||
|
|
|
||
|
|
def run(self) -> None:
|
||
|
|
if self.quiet:
|
||
|
|
disable_progress_bars()
|
||
|
|
with warnings.catch_warnings():
|
||
|
|
warnings.simplefilter("ignore")
|
||
|
|
print(self._upload())
|
||
|
|
enable_progress_bars()
|
||
|
|
else:
|
||
|
|
logging.set_verbosity_info()
|
||
|
|
print(self._upload())
|
||
|
|
logging.set_verbosity_warning()
|
||
|
|
|
||
|
|
def _upload(self) -> str:
|
||
|
|
if os.path.isfile(self.local_path):
|
||
|
|
if self.include is not None and len(self.include) > 0:
|
||
|
|
warnings.warn("Ignoring `--include` since a single file is uploaded.")
|
||
|
|
if self.exclude is not None and len(self.exclude) > 0:
|
||
|
|
warnings.warn("Ignoring `--exclude` since a single file is uploaded.")
|
||
|
|
if self.delete is not None and len(self.delete) > 0:
|
||
|
|
warnings.warn("Ignoring `--delete` since a single file is uploaded.")
|
||
|
|
|
||
|
|
if not HF_HUB_ENABLE_HF_TRANSFER:
|
||
|
|
logger.info(
|
||
|
|
"Consider using `hf_transfer` for faster uploads. This solution comes with some limitations. See"
|
||
|
|
" https://huggingface.co/docs/huggingface_hub/hf_transfer for more details."
|
||
|
|
)
|
||
|
|
|
||
|
|
# Schedule commits if `every` is set
|
||
|
|
if self.every is not None:
|
||
|
|
if os.path.isfile(self.local_path):
|
||
|
|
# If file => watch entire folder + use allow_patterns
|
||
|
|
folder_path = os.path.dirname(self.local_path)
|
||
|
|
path_in_repo = (
|
||
|
|
self.path_in_repo[: -len(self.local_path)] # remove filename from path_in_repo
|
||
|
|
if self.path_in_repo.endswith(self.local_path)
|
||
|
|
else self.path_in_repo
|
||
|
|
)
|
||
|
|
allow_patterns = [self.local_path]
|
||
|
|
ignore_patterns = []
|
||
|
|
else:
|
||
|
|
folder_path = self.local_path
|
||
|
|
path_in_repo = self.path_in_repo
|
||
|
|
allow_patterns = self.include or []
|
||
|
|
ignore_patterns = self.exclude or []
|
||
|
|
if self.delete is not None and len(self.delete) > 0:
|
||
|
|
warnings.warn("Ignoring `--delete` when uploading with scheduled commits.")
|
||
|
|
|
||
|
|
scheduler = CommitScheduler(
|
||
|
|
folder_path=folder_path,
|
||
|
|
repo_id=self.repo_id,
|
||
|
|
repo_type=self.repo_type,
|
||
|
|
revision=self.revision,
|
||
|
|
allow_patterns=allow_patterns,
|
||
|
|
ignore_patterns=ignore_patterns,
|
||
|
|
path_in_repo=path_in_repo,
|
||
|
|
private=self.private,
|
||
|
|
every=self.every,
|
||
|
|
hf_api=self.api,
|
||
|
|
)
|
||
|
|
print(f"Scheduling commits every {self.every} minutes to {scheduler.repo_id}.")
|
||
|
|
try: # Block main thread until KeyboardInterrupt
|
||
|
|
while True:
|
||
|
|
time.sleep(100)
|
||
|
|
except KeyboardInterrupt:
|
||
|
|
scheduler.stop()
|
||
|
|
return "Stopped scheduled commits."
|
||
|
|
|
||
|
|
# Otherwise, create repo and proceed with the upload
|
||
|
|
if not os.path.isfile(self.local_path) and not os.path.isdir(self.local_path):
|
||
|
|
raise FileNotFoundError(f"No such file or directory: '{self.local_path}'.")
|
||
|
|
repo_id = self.api.create_repo(
|
||
|
|
repo_id=self.repo_id,
|
||
|
|
repo_type=self.repo_type,
|
||
|
|
exist_ok=True,
|
||
|
|
private=self.private,
|
||
|
|
space_sdk="gradio" if self.repo_type == "space" else None,
|
||
|
|
# ^ We don't want it to fail when uploading to a Space => let's set Gradio by default.
|
||
|
|
# ^ I'd rather not add CLI args to set it explicitly as we already have `huggingface-cli repo create` for that.
|
||
|
|
).repo_id
|
||
|
|
|
||
|
|
# Check if branch already exists and if not, create it
|
||
|
|
if self.revision is not None and not self.create_pr:
|
||
|
|
try:
|
||
|
|
self.api.repo_info(repo_id=repo_id, repo_type=self.repo_type, revision=self.revision)
|
||
|
|
except RevisionNotFoundError:
|
||
|
|
logger.info(f"Branch '{self.revision}' not found. Creating it...")
|
||
|
|
self.api.create_branch(repo_id=repo_id, repo_type=self.repo_type, branch=self.revision, exist_ok=True)
|
||
|
|
# ^ `exist_ok=True` to avoid race concurrency issues
|
||
|
|
|
||
|
|
# File-based upload
|
||
|
|
if os.path.isfile(self.local_path):
|
||
|
|
return self.api.upload_file(
|
||
|
|
path_or_fileobj=self.local_path,
|
||
|
|
path_in_repo=self.path_in_repo,
|
||
|
|
repo_id=repo_id,
|
||
|
|
repo_type=self.repo_type,
|
||
|
|
revision=self.revision,
|
||
|
|
commit_message=self.commit_message,
|
||
|
|
commit_description=self.commit_description,
|
||
|
|
create_pr=self.create_pr,
|
||
|
|
)
|
||
|
|
|
||
|
|
# Folder-based upload
|
||
|
|
else:
|
||
|
|
return self.api.upload_folder(
|
||
|
|
folder_path=self.local_path,
|
||
|
|
path_in_repo=self.path_in_repo,
|
||
|
|
repo_id=repo_id,
|
||
|
|
repo_type=self.repo_type,
|
||
|
|
revision=self.revision,
|
||
|
|
commit_message=self.commit_message,
|
||
|
|
commit_description=self.commit_description,
|
||
|
|
create_pr=self.create_pr,
|
||
|
|
allow_patterns=self.include,
|
||
|
|
ignore_patterns=self.exclude,
|
||
|
|
delete_patterns=self.delete,
|
||
|
|
)
|