[Storage] Fix copy monuts for file with s3 bucket url (skypilot-org#1457)

* test file download with s3

* fix test

* fix storage file mounts

* format

* remove mkdir for `make_sync_dir_command`

Author: Michaelvll

examples/managed_spot_with_storage.yaml

@@ -52,3 +52,6 @@ run: |
 
   echo hi >> /tmp/workdir/new_file
   ls -al /tmp/workdir
+
+  cat ~/tmpfile
+  cat ~/a/b/c/tmpfile

examples/using_file_mounts.yaml

@@ -69,6 +69,7 @@ file_mounts:
   ~/data/train-00001-of-01024: gs://cloud-tpu-test-datasets/fake_imagenet/train-00001-of-01024
 
   /s3-data-test: s3://fah-public-data-covid19-cryptic-pockets/human/il6/PROJ14534/RUN999/CLONE0/results0
+  /s3-data-file: s3://fah-public-data-covid19-cryptic-pockets/human/il6/PROJ14534/RUN999/CLONE0/results0/frame0.xtc
   # /test-my-gcs: gs://cloud-storage-test-zhwu-2
 
   # If a source path points to a "directory", its contents will be recursively
@@ -113,6 +114,7 @@ run: |
   ls -l ~/data/train-00001-of-01024
 
   ls -l /s3-data-test
+  cat /s3-data-file > /dev/null
 
   # ls -lH /data/fake_imagenet | head -n10
 

sky/backends/cloud_vm_ray_backend.py

@@ -2975,8 +2975,8 @@ def _execute_file_mounts(self, handle: ResourceHandle,
                 sync = storage.make_sync_file_command(source=src,
                                                       destination=wrapped_dst)
                 # It is a file so make sure *its parent dir* exists.
-                mkdir_for_wrapped_dst = \
-                    f'mkdir -p {os.path.dirname(wrapped_dst)}'
+                mkdir_for_wrapped_dst = (
+                    f'mkdir -p {os.path.dirname(wrapped_dst)}')
 
             download_target_commands = [
                 # Ensure sync can write to wrapped_dst (e.g., '/data/').

sky/cloud_stores.py

@@ -70,18 +70,16 @@ def make_sync_dir_command(self, source: str, destination: str) -> str:
         # AWS Sync by default uses 10 threads to upload files to the bucket.
         # To increase parallelism, modify max_concurrent_requests in your
         # aws config file (Default path: ~/.aws/config).
-        download_via_awscli = f'mkdir -p {destination} && \
-                                aws s3 sync --no-follow-symlinks ' \
-                              f'{source} {destination}'
+        download_via_awscli = ('aws s3 sync --no-follow-symlinks '
+                               f'{source} {destination}')
 
         all_commands = list(self._GET_AWSCLI)
         all_commands.append(download_via_awscli)
         return ' && '.join(all_commands)
 
     def make_sync_file_command(self, source: str, destination: str) -> str:
         """Downloads a file using AWS CLI."""
-        download_via_awscli = (f'mkdir -p {destination} &&'
-                               f'aws s3 cp {source} {destination}')
+        download_via_awscli = f'aws s3 cp {source} {destination}'
 
         all_commands = list(self._GET_AWSCLI)
         all_commands.append(download_via_awscli)