Merge pull request #737 from martindurant/put_single

Put files under target directory if it exists

Author: martindurant

fsspec/asyn.py

@@ -378,7 +378,9 @@ async def _put(
             lpath = make_path_posix(lpath)
         fs = LocalFileSystem()
         lpaths = fs.expand_path(lpath, recursive=recursive)
-        rpaths = other_paths(lpaths, rpath)
+        rpaths = other_paths(
+            lpaths, rpath, exists=isinstance(rpath, str) and await self._isdir(rpath)
+        )
 
         is_dir = {l: os.path.isdir(l) for l in lpaths}
         rdirs = [r for l, r in zip(lpaths, rpaths) if is_dir[l]]

fsspec/implementations/tests/test_memory.py

@@ -1,3 +1,5 @@
+import os
+
 import pytest
 
 
@@ -19,6 +21,17 @@ def test_strip(m):
     assert m._strip_protocol("/b/c/") == "/b/c"
 
 
+def test_put_single(m, tmpdir):
+    fn = os.path.join(str(tmpdir), "dir")
+    os.mkdir(fn)
+    open(os.path.join(fn, "abc"), "w").write("text")
+    m.put(fn, "/test")  # no-op, no files
+    assert not m.exists("/test/abc")
+    assert not m.exists("/test/dir")
+    m.put(fn + "/", "/test", recursive=True)
+    assert m.cat("/test/dir/abc") == b"text"
+
+
 def test_ls(m):
     m.mkdir("/dir")
     m.mkdir("/dir/dir1")

fsspec/spec.py

@@ -802,7 +802,9 @@ def put(self, lpath, rpath, recursive=False, callback=_DEFAULT_CALLBACK, **kwarg
             lpath = make_path_posix(lpath)
         fs = LocalFileSystem()
         lpaths = fs.expand_path(lpath, recursive=recursive)
-        rpaths = other_paths(lpaths, rpath)
+        rpaths = other_paths(
+            lpaths, rpath, exists=isinstance(rpath, str) and self.isdir(rpath)
+        )
 
         callback.set_size(len(rpaths))
         for lpath, rpath in callback.wrap(zip(lpaths, rpaths)):

fsspec/tests/test_spec.py

@@ -462,7 +462,7 @@ def imitate_transfer(size, chunk, *, file=True):
     if file:
         # The reason that there is a relative_update(0) at the
         # end is that, we don't have an early exit on the
-        # impleementations of get_file/put_file so it needs to
+        # implementations of get_file/put_file so it needs to
         # go through the callback to get catch by the while's
         # condition and then it will stop the transfer.
         events.append(("relative_update", 0))

fsspec/utils.py

@@ -335,7 +335,7 @@ def common_prefix(paths):
     return "/".join(parts[0][:i])
 
 
-def other_paths(paths, path2, is_dir=None):
+def other_paths(paths, path2, is_dir=None, exists=False):
     """In bulk file operations, construct a new file tree from a list of files
 
     Parameters
@@ -349,6 +349,9 @@ def other_paths(paths, path2, is_dir=None):
         For the special case where the input in one element, whether to regard the value
         as the target path, or as a directory to put a file path within. If None, a
         directory is inferred if the path ends in '/'
+    exists: bool (optional)
+        For a str destination, it is already exists (and is a dir), files should
+        end up inside.
 
     Returns
     -------
@@ -359,6 +362,8 @@ def other_paths(paths, path2, is_dir=None):
         path2 = path2.rstrip("/")
         if len(paths) > 1:
             cp = common_prefix(paths)
+            if exists:
+                cp = cp.rsplit("/", 1)[0]
             path2 = [p.replace(cp, path2, 1) for p in paths]
         else:
             if is_dir: