592 lines
21 KiB
Python
592 lines
21 KiB
Python
from hashlib import md5
|
|
from itertools import product
|
|
|
|
import pytest
|
|
|
|
from fsspec.tests.abstract.common import GLOB_EDGE_CASES_TESTS
|
|
|
|
|
|
class AbstractPutTests:
|
|
def test_put_file_to_existing_directory(
|
|
self,
|
|
fs,
|
|
fs_join,
|
|
fs_target,
|
|
local_join,
|
|
local_bulk_operations_scenario_0,
|
|
supports_empty_directories,
|
|
):
|
|
# Copy scenario 1a
|
|
source = local_bulk_operations_scenario_0
|
|
|
|
target = fs_target
|
|
fs.mkdir(target)
|
|
if not supports_empty_directories:
|
|
# Force target directory to exist by adding a dummy file
|
|
fs.touch(fs_join(target, "dummy"))
|
|
assert fs.isdir(target)
|
|
|
|
target_file2 = fs_join(target, "file2")
|
|
target_subfile1 = fs_join(target, "subfile1")
|
|
|
|
# Copy from source directory
|
|
fs.put(local_join(source, "file2"), target)
|
|
assert fs.isfile(target_file2)
|
|
|
|
# Copy from sub directory
|
|
fs.put(local_join(source, "subdir", "subfile1"), target)
|
|
assert fs.isfile(target_subfile1)
|
|
|
|
# Remove copied files
|
|
fs.rm([target_file2, target_subfile1])
|
|
assert not fs.exists(target_file2)
|
|
assert not fs.exists(target_subfile1)
|
|
|
|
# Repeat with trailing slash on target
|
|
fs.put(local_join(source, "file2"), target + "/")
|
|
assert fs.isdir(target)
|
|
assert fs.isfile(target_file2)
|
|
|
|
fs.put(local_join(source, "subdir", "subfile1"), target + "/")
|
|
assert fs.isfile(target_subfile1)
|
|
|
|
def test_put_file_to_new_directory(
|
|
self, fs, fs_join, fs_target, local_join, local_bulk_operations_scenario_0
|
|
):
|
|
# Copy scenario 1b
|
|
source = local_bulk_operations_scenario_0
|
|
|
|
target = fs_target
|
|
fs.mkdir(target)
|
|
|
|
fs.put(
|
|
local_join(source, "subdir", "subfile1"), fs_join(target, "newdir/")
|
|
) # Note trailing slash
|
|
assert fs.isdir(target)
|
|
assert fs.isdir(fs_join(target, "newdir"))
|
|
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
|
|
|
|
def test_put_file_to_file_in_existing_directory(
|
|
self,
|
|
fs,
|
|
fs_join,
|
|
fs_target,
|
|
local_join,
|
|
supports_empty_directories,
|
|
local_bulk_operations_scenario_0,
|
|
):
|
|
# Copy scenario 1c
|
|
source = local_bulk_operations_scenario_0
|
|
|
|
target = fs_target
|
|
fs.mkdir(target)
|
|
if not supports_empty_directories:
|
|
# Force target directory to exist by adding a dummy file
|
|
fs.touch(fs_join(target, "dummy"))
|
|
assert fs.isdir(target)
|
|
|
|
fs.put(local_join(source, "subdir", "subfile1"), fs_join(target, "newfile"))
|
|
assert fs.isfile(fs_join(target, "newfile"))
|
|
|
|
def test_put_file_to_file_in_new_directory(
|
|
self, fs, fs_join, fs_target, local_join, local_bulk_operations_scenario_0
|
|
):
|
|
# Copy scenario 1d
|
|
source = local_bulk_operations_scenario_0
|
|
|
|
target = fs_target
|
|
fs.mkdir(target)
|
|
|
|
fs.put(
|
|
local_join(source, "subdir", "subfile1"),
|
|
fs_join(target, "newdir", "newfile"),
|
|
)
|
|
assert fs.isdir(fs_join(target, "newdir"))
|
|
assert fs.isfile(fs_join(target, "newdir", "newfile"))
|
|
|
|
def test_put_directory_to_existing_directory(
|
|
self,
|
|
fs,
|
|
fs_join,
|
|
fs_target,
|
|
local_bulk_operations_scenario_0,
|
|
supports_empty_directories,
|
|
):
|
|
# Copy scenario 1e
|
|
source = local_bulk_operations_scenario_0
|
|
|
|
target = fs_target
|
|
fs.mkdir(target)
|
|
if not supports_empty_directories:
|
|
# Force target directory to exist by adding a dummy file
|
|
dummy = fs_join(target, "dummy")
|
|
fs.touch(dummy)
|
|
assert fs.isdir(target)
|
|
|
|
for source_slash, target_slash in zip([False, True], [False, True]):
|
|
s = fs_join(source, "subdir")
|
|
if source_slash:
|
|
s += "/"
|
|
t = target + "/" if target_slash else target
|
|
|
|
# Without recursive does nothing
|
|
fs.put(s, t)
|
|
assert fs.ls(target, detail=False) == (
|
|
[] if supports_empty_directories else [dummy]
|
|
)
|
|
|
|
# With recursive
|
|
fs.put(s, t, recursive=True)
|
|
if source_slash:
|
|
assert fs.isfile(fs_join(target, "subfile1"))
|
|
assert fs.isfile(fs_join(target, "subfile2"))
|
|
assert fs.isdir(fs_join(target, "nesteddir"))
|
|
assert fs.isfile(fs_join(target, "nesteddir", "nestedfile"))
|
|
assert not fs.exists(fs_join(target, "subdir"))
|
|
|
|
fs.rm(
|
|
[
|
|
fs_join(target, "subfile1"),
|
|
fs_join(target, "subfile2"),
|
|
fs_join(target, "nesteddir"),
|
|
],
|
|
recursive=True,
|
|
)
|
|
else:
|
|
assert fs.isdir(fs_join(target, "subdir"))
|
|
assert fs.isfile(fs_join(target, "subdir", "subfile1"))
|
|
assert fs.isfile(fs_join(target, "subdir", "subfile2"))
|
|
assert fs.isdir(fs_join(target, "subdir", "nesteddir"))
|
|
assert fs.isfile(fs_join(target, "subdir", "nesteddir", "nestedfile"))
|
|
|
|
fs.rm(fs_join(target, "subdir"), recursive=True)
|
|
assert fs.ls(target, detail=False) == (
|
|
[] if supports_empty_directories else [dummy]
|
|
)
|
|
|
|
# Limit recursive by maxdepth
|
|
fs.put(s, t, recursive=True, maxdepth=1)
|
|
if source_slash:
|
|
assert fs.isfile(fs_join(target, "subfile1"))
|
|
assert fs.isfile(fs_join(target, "subfile2"))
|
|
assert not fs.exists(fs_join(target, "nesteddir"))
|
|
assert not fs.exists(fs_join(target, "subdir"))
|
|
|
|
fs.rm(
|
|
[
|
|
fs_join(target, "subfile1"),
|
|
fs_join(target, "subfile2"),
|
|
],
|
|
recursive=True,
|
|
)
|
|
else:
|
|
assert fs.isdir(fs_join(target, "subdir"))
|
|
assert fs.isfile(fs_join(target, "subdir", "subfile1"))
|
|
assert fs.isfile(fs_join(target, "subdir", "subfile2"))
|
|
assert not fs.exists(fs_join(target, "subdir", "nesteddir"))
|
|
|
|
fs.rm(fs_join(target, "subdir"), recursive=True)
|
|
assert fs.ls(target, detail=False) == (
|
|
[] if supports_empty_directories else [dummy]
|
|
)
|
|
|
|
def test_put_directory_to_new_directory(
|
|
self,
|
|
fs,
|
|
fs_join,
|
|
fs_target,
|
|
local_bulk_operations_scenario_0,
|
|
supports_empty_directories,
|
|
):
|
|
# Copy scenario 1f
|
|
source = local_bulk_operations_scenario_0
|
|
|
|
target = fs_target
|
|
fs.mkdir(target)
|
|
|
|
for source_slash, target_slash in zip([False, True], [False, True]):
|
|
s = fs_join(source, "subdir")
|
|
if source_slash:
|
|
s += "/"
|
|
t = fs_join(target, "newdir")
|
|
if target_slash:
|
|
t += "/"
|
|
|
|
# Without recursive does nothing
|
|
fs.put(s, t)
|
|
if supports_empty_directories:
|
|
assert fs.ls(target) == []
|
|
else:
|
|
with pytest.raises(FileNotFoundError):
|
|
fs.ls(target)
|
|
|
|
# With recursive
|
|
fs.put(s, t, recursive=True)
|
|
assert fs.isdir(fs_join(target, "newdir"))
|
|
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
|
|
assert fs.isfile(fs_join(target, "newdir", "subfile2"))
|
|
assert fs.isdir(fs_join(target, "newdir", "nesteddir"))
|
|
assert fs.isfile(fs_join(target, "newdir", "nesteddir", "nestedfile"))
|
|
assert not fs.exists(fs_join(target, "subdir"))
|
|
|
|
fs.rm(fs_join(target, "newdir"), recursive=True)
|
|
assert not fs.exists(fs_join(target, "newdir"))
|
|
|
|
# Limit recursive by maxdepth
|
|
fs.put(s, t, recursive=True, maxdepth=1)
|
|
assert fs.isdir(fs_join(target, "newdir"))
|
|
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
|
|
assert fs.isfile(fs_join(target, "newdir", "subfile2"))
|
|
assert not fs.exists(fs_join(target, "newdir", "nesteddir"))
|
|
assert not fs.exists(fs_join(target, "subdir"))
|
|
|
|
fs.rm(fs_join(target, "newdir"), recursive=True)
|
|
assert not fs.exists(fs_join(target, "newdir"))
|
|
|
|
def test_put_glob_to_existing_directory(
|
|
self,
|
|
fs,
|
|
fs_join,
|
|
fs_target,
|
|
local_join,
|
|
supports_empty_directories,
|
|
local_bulk_operations_scenario_0,
|
|
):
|
|
# Copy scenario 1g
|
|
source = local_bulk_operations_scenario_0
|
|
|
|
target = fs_target
|
|
fs.mkdir(target)
|
|
if not supports_empty_directories:
|
|
# Force target directory to exist by adding a dummy file
|
|
dummy = fs_join(target, "dummy")
|
|
fs.touch(dummy)
|
|
assert fs.isdir(target)
|
|
|
|
for target_slash in [False, True]:
|
|
t = target + "/" if target_slash else target
|
|
|
|
# Without recursive
|
|
fs.put(local_join(source, "subdir", "*"), t)
|
|
assert fs.isfile(fs_join(target, "subfile1"))
|
|
assert fs.isfile(fs_join(target, "subfile2"))
|
|
assert not fs.isdir(fs_join(target, "nesteddir"))
|
|
assert not fs.exists(fs_join(target, "nesteddir", "nestedfile"))
|
|
assert not fs.exists(fs_join(target, "subdir"))
|
|
|
|
fs.rm(
|
|
[
|
|
fs_join(target, "subfile1"),
|
|
fs_join(target, "subfile2"),
|
|
],
|
|
recursive=True,
|
|
)
|
|
assert fs.ls(target, detail=False) == (
|
|
[] if supports_empty_directories else [dummy]
|
|
)
|
|
|
|
# With recursive
|
|
for glob, recursive in zip(["*", "**"], [True, False]):
|
|
fs.put(local_join(source, "subdir", glob), t, recursive=recursive)
|
|
assert fs.isfile(fs_join(target, "subfile1"))
|
|
assert fs.isfile(fs_join(target, "subfile2"))
|
|
assert fs.isdir(fs_join(target, "nesteddir"))
|
|
assert fs.isfile(fs_join(target, "nesteddir", "nestedfile"))
|
|
assert not fs.exists(fs_join(target, "subdir"))
|
|
|
|
fs.rm(
|
|
[
|
|
fs_join(target, "subfile1"),
|
|
fs_join(target, "subfile2"),
|
|
fs_join(target, "nesteddir"),
|
|
],
|
|
recursive=True,
|
|
)
|
|
assert fs.ls(target, detail=False) == (
|
|
[] if supports_empty_directories else [dummy]
|
|
)
|
|
|
|
# Limit recursive by maxdepth
|
|
fs.put(
|
|
local_join(source, "subdir", glob),
|
|
t,
|
|
recursive=recursive,
|
|
maxdepth=1,
|
|
)
|
|
assert fs.isfile(fs_join(target, "subfile1"))
|
|
assert fs.isfile(fs_join(target, "subfile2"))
|
|
assert not fs.exists(fs_join(target, "nesteddir"))
|
|
assert not fs.exists(fs_join(target, "subdir"))
|
|
|
|
fs.rm(
|
|
[
|
|
fs_join(target, "subfile1"),
|
|
fs_join(target, "subfile2"),
|
|
],
|
|
recursive=True,
|
|
)
|
|
assert fs.ls(target, detail=False) == (
|
|
[] if supports_empty_directories else [dummy]
|
|
)
|
|
|
|
def test_put_glob_to_new_directory(
|
|
self, fs, fs_join, fs_target, local_join, local_bulk_operations_scenario_0
|
|
):
|
|
# Copy scenario 1h
|
|
source = local_bulk_operations_scenario_0
|
|
|
|
target = fs_target
|
|
fs.mkdir(target)
|
|
|
|
for target_slash in [False, True]:
|
|
t = fs_join(target, "newdir")
|
|
if target_slash:
|
|
t += "/"
|
|
|
|
# Without recursive
|
|
fs.put(local_join(source, "subdir", "*"), t)
|
|
assert fs.isdir(fs_join(target, "newdir"))
|
|
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
|
|
assert fs.isfile(fs_join(target, "newdir", "subfile2"))
|
|
assert not fs.exists(fs_join(target, "newdir", "nesteddir"))
|
|
assert not fs.exists(fs_join(target, "newdir", "nesteddir", "nestedfile"))
|
|
assert not fs.exists(fs_join(target, "subdir"))
|
|
assert not fs.exists(fs_join(target, "newdir", "subdir"))
|
|
|
|
fs.rm(fs_join(target, "newdir"), recursive=True)
|
|
assert not fs.exists(fs_join(target, "newdir"))
|
|
|
|
# With recursive
|
|
for glob, recursive in zip(["*", "**"], [True, False]):
|
|
fs.put(local_join(source, "subdir", glob), t, recursive=recursive)
|
|
assert fs.isdir(fs_join(target, "newdir"))
|
|
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
|
|
assert fs.isfile(fs_join(target, "newdir", "subfile2"))
|
|
assert fs.isdir(fs_join(target, "newdir", "nesteddir"))
|
|
assert fs.isfile(fs_join(target, "newdir", "nesteddir", "nestedfile"))
|
|
assert not fs.exists(fs_join(target, "subdir"))
|
|
assert not fs.exists(fs_join(target, "newdir", "subdir"))
|
|
|
|
fs.rm(fs_join(target, "newdir"), recursive=True)
|
|
assert not fs.exists(fs_join(target, "newdir"))
|
|
|
|
# Limit recursive by maxdepth
|
|
fs.put(
|
|
local_join(source, "subdir", glob),
|
|
t,
|
|
recursive=recursive,
|
|
maxdepth=1,
|
|
)
|
|
assert fs.isdir(fs_join(target, "newdir"))
|
|
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
|
|
assert fs.isfile(fs_join(target, "newdir", "subfile2"))
|
|
assert not fs.exists(fs_join(target, "newdir", "nesteddir"))
|
|
assert not fs.exists(fs_join(target, "subdir"))
|
|
assert not fs.exists(fs_join(target, "newdir", "subdir"))
|
|
|
|
fs.rm(fs_join(target, "newdir"), recursive=True)
|
|
assert not fs.exists(fs_join(target, "newdir"))
|
|
|
|
@pytest.mark.parametrize(
|
|
GLOB_EDGE_CASES_TESTS["argnames"],
|
|
GLOB_EDGE_CASES_TESTS["argvalues"],
|
|
)
|
|
def test_put_glob_edge_cases(
|
|
self,
|
|
path,
|
|
recursive,
|
|
maxdepth,
|
|
expected,
|
|
fs,
|
|
fs_join,
|
|
fs_target,
|
|
local_glob_edge_cases_files,
|
|
local_join,
|
|
fs_sanitize_path,
|
|
):
|
|
# Copy scenario 1g
|
|
source = local_glob_edge_cases_files
|
|
|
|
target = fs_target
|
|
|
|
for new_dir, target_slash in product([True, False], [True, False]):
|
|
fs.mkdir(target)
|
|
|
|
t = fs_join(target, "newdir") if new_dir else target
|
|
t = t + "/" if target_slash else t
|
|
|
|
fs.put(local_join(source, path), t, recursive=recursive, maxdepth=maxdepth)
|
|
|
|
output = fs.find(target)
|
|
if new_dir:
|
|
prefixed_expected = [
|
|
fs_sanitize_path(fs_join(target, "newdir", p)) for p in expected
|
|
]
|
|
else:
|
|
prefixed_expected = [
|
|
fs_sanitize_path(fs_join(target, p)) for p in expected
|
|
]
|
|
assert sorted(output) == sorted(prefixed_expected)
|
|
|
|
try:
|
|
fs.rm(target, recursive=True)
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
def test_put_list_of_files_to_existing_directory(
|
|
self,
|
|
fs,
|
|
fs_join,
|
|
fs_target,
|
|
local_join,
|
|
local_bulk_operations_scenario_0,
|
|
supports_empty_directories,
|
|
):
|
|
# Copy scenario 2a
|
|
source = local_bulk_operations_scenario_0
|
|
|
|
target = fs_target
|
|
fs.mkdir(target)
|
|
if not supports_empty_directories:
|
|
# Force target directory to exist by adding a dummy file
|
|
dummy = fs_join(target, "dummy")
|
|
fs.touch(dummy)
|
|
assert fs.isdir(target)
|
|
|
|
source_files = [
|
|
local_join(source, "file1"),
|
|
local_join(source, "file2"),
|
|
local_join(source, "subdir", "subfile1"),
|
|
]
|
|
|
|
for target_slash in [False, True]:
|
|
t = target + "/" if target_slash else target
|
|
|
|
fs.put(source_files, t)
|
|
assert fs.isfile(fs_join(target, "file1"))
|
|
assert fs.isfile(fs_join(target, "file2"))
|
|
assert fs.isfile(fs_join(target, "subfile1"))
|
|
|
|
fs.rm(
|
|
[
|
|
fs_join(target, "file1"),
|
|
fs_join(target, "file2"),
|
|
fs_join(target, "subfile1"),
|
|
],
|
|
recursive=True,
|
|
)
|
|
assert fs.ls(target, detail=False) == (
|
|
[] if supports_empty_directories else [dummy]
|
|
)
|
|
|
|
def test_put_list_of_files_to_new_directory(
|
|
self, fs, fs_join, fs_target, local_join, local_bulk_operations_scenario_0
|
|
):
|
|
# Copy scenario 2b
|
|
source = local_bulk_operations_scenario_0
|
|
|
|
target = fs_target
|
|
fs.mkdir(target)
|
|
|
|
source_files = [
|
|
local_join(source, "file1"),
|
|
local_join(source, "file2"),
|
|
local_join(source, "subdir", "subfile1"),
|
|
]
|
|
|
|
fs.put(source_files, fs_join(target, "newdir") + "/") # Note trailing slash
|
|
assert fs.isdir(fs_join(target, "newdir"))
|
|
assert fs.isfile(fs_join(target, "newdir", "file1"))
|
|
assert fs.isfile(fs_join(target, "newdir", "file2"))
|
|
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
|
|
|
|
def test_put_directory_recursive(
|
|
self, fs, fs_join, fs_target, local_fs, local_join, local_path
|
|
):
|
|
# https://github.com/fsspec/filesystem_spec/issues/1062
|
|
# Recursive cp/get/put of source directory into non-existent target directory.
|
|
src = local_join(local_path, "src")
|
|
src_file = local_join(src, "file")
|
|
local_fs.mkdir(src)
|
|
local_fs.touch(src_file)
|
|
|
|
target = fs_target
|
|
|
|
# put without slash
|
|
assert not fs.exists(target)
|
|
for loop in range(2):
|
|
fs.put(src, target, recursive=True)
|
|
assert fs.isdir(target)
|
|
|
|
if loop == 0:
|
|
assert fs.isfile(fs_join(target, "file"))
|
|
assert not fs.exists(fs_join(target, "src"))
|
|
else:
|
|
assert fs.isfile(fs_join(target, "file"))
|
|
assert fs.isdir(fs_join(target, "src"))
|
|
assert fs.isfile(fs_join(target, "src", "file"))
|
|
|
|
fs.rm(target, recursive=True)
|
|
|
|
# put with slash
|
|
assert not fs.exists(target)
|
|
for loop in range(2):
|
|
fs.put(src + "/", target, recursive=True)
|
|
assert fs.isdir(target)
|
|
assert fs.isfile(fs_join(target, "file"))
|
|
assert not fs.exists(fs_join(target, "src"))
|
|
|
|
def test_put_directory_without_files_with_same_name_prefix(
|
|
self,
|
|
fs,
|
|
fs_join,
|
|
fs_target,
|
|
local_join,
|
|
local_dir_and_file_with_same_name_prefix,
|
|
supports_empty_directories,
|
|
):
|
|
# Create the test dirs
|
|
source = local_dir_and_file_with_same_name_prefix
|
|
target = fs_target
|
|
|
|
# Test without glob
|
|
fs.put(local_join(source, "subdir"), fs_target, recursive=True)
|
|
|
|
assert fs.isfile(fs_join(fs_target, "subfile.txt"))
|
|
assert not fs.isfile(fs_join(fs_target, "subdir.txt"))
|
|
|
|
fs.rm([fs_join(target, "subfile.txt")])
|
|
if supports_empty_directories:
|
|
assert fs.ls(target) == []
|
|
else:
|
|
assert not fs.exists(target)
|
|
|
|
# Test with glob
|
|
fs.put(local_join(source, "subdir*"), fs_target, recursive=True)
|
|
|
|
assert fs.isdir(fs_join(fs_target, "subdir"))
|
|
assert fs.isfile(fs_join(fs_target, "subdir", "subfile.txt"))
|
|
assert fs.isfile(fs_join(fs_target, "subdir.txt"))
|
|
|
|
def test_copy_with_source_and_destination_as_list(
|
|
self, fs, fs_target, fs_join, local_join, local_10_files_with_hashed_names
|
|
):
|
|
# Create the test dir
|
|
source = local_10_files_with_hashed_names
|
|
target = fs_target
|
|
|
|
# Create list of files for source and destination
|
|
source_files = []
|
|
destination_files = []
|
|
for i in range(10):
|
|
hashed_i = md5(str(i).encode("utf-8")).hexdigest()
|
|
source_files.append(local_join(source, f"{hashed_i}.txt"))
|
|
destination_files.append(fs_join(target, f"{hashed_i}.txt"))
|
|
|
|
# Copy and assert order was kept
|
|
fs.put(lpath=source_files, rpath=destination_files)
|
|
|
|
for i in range(10):
|
|
file_content = fs.cat(destination_files[i]).decode("utf-8")
|
|
assert file_content == str(i)
|