[Storage] Smoke tests for Storage mounting (skypilot-org#1445)

* Set rename_dir_lim for gcsfuse

* Add support for list of sources for Storage

* fix demo yaml

* tests

* lint

* lint

* test

* add validation

* address zhwu comments

* add error on basename conflicts

* use gsutil cp -n instead of gsutil rsync

* lint

* fix name

* parallelize gsutil rsync

* parallelize aws s3 rsync

* lint

* address comments

* refactor

* lint

* address comments

* Add storage mounting tests

* lint

* comments

* comments

* lint

* add setup commands

* remove teardown

* lint

Author: romilbhardwaj

tests/test_smoke.py

@@ -10,6 +10,7 @@
 import uuid
 
 import colorama
+import jinja2
 import pytest
 
 import sky
@@ -31,6 +32,12 @@
 # id.
 test_id = str(uuid.uuid4())[-2:]
 
+storage_setup_commands = [
+    'touch ~/tmpfile', 'mkdir -p ~/tmp-workdir',
+    'touch ~/tmp-workdir/tmp\ file', 'touch ~/tmp-workdir/foo',
+    'ln -f -s ~/tmp-workdir/ ~/tmp-workdir/circle-link'
+]
+
 
 class Test(NamedTuple):
     name: str
@@ -327,23 +334,51 @@ def test_env_check():
 # ---------- file_mounts ----------
 def test_file_mounts():
     name = _get_cluster_name()
+    test_commands = [
+        *storage_setup_commands,
+        f'sky launch -y -c {name} examples/using_file_mounts.yaml',
+        f'sky logs {name} 1 --status',  # Ensure the job succeeded.
+    ]
     test = Test(
         'using_file_mounts',
-        [
-            'touch ~/tmpfile',
-            'mkdir -p ~/tmp-workdir',
-            'touch ~/tmp-workdir/tmp\ file',
-            'touch ~/tmp-workdir/foo',
-            'ln -f -s ~/tmp-workdir/ ~/tmp-workdir/circle-link',
-            f'sky launch -y -c {name} examples/using_file_mounts.yaml',
-            f'sky logs {name} 1 --status',  # Ensure the job succeeded.
-        ],
+        test_commands,
         f'sky down -y {name}',
         timeout=20 * 60,  # 20 mins
     )
     run_one_test(test)
 
 
+# ---------- storage ----------
+def test_storage_mounts():
+    name = _get_cluster_name()
+    storage_name = f'sky-test-{int(time.time())}'
+    template_str = pathlib.Path(
+        'tests/test_yamls/test_storage_mounting.yaml').read_text()
+    template = jinja2.Template(template_str)
+    content = template.render(storage_name=storage_name)
+    with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w') as f:
+        f.write(content)
+        f.flush()
+        file_path = f.name
+        test_commands = [
+            *storage_setup_commands,
+            f'sky launch -y -c {name}-aws --cloud aws {file_path}',
+            f'sky logs {name}-aws 1 --status',  # Ensure job succeeded.
+            f'aws s3 ls {storage_name}/hello.txt',
+            f'sky storage delete {storage_name}',  # Prepare for next cloud
+            f'sky launch -y -c {name}-gcp --cloud gcp {file_path}',
+            f'sky logs {name}-gcp 1 --status',  # Ensure job succeeded.
+            f'gsutil ls gs://{storage_name}/hello.txt',
+        ]
+        test = Test(
+            'storage_mounts',
+            test_commands,
+            f'sky down -y {name}-aws {name}-gcp; sky storage delete {storage_name}',
+            timeout=20 * 60,  # 20 mins
+        )
+        run_one_test(test)
+
+
 # ---------- CLI logs ----------
 def test_logs():
     name = _get_cluster_name()
@@ -921,7 +956,7 @@ def test_spot_cancellation():
              '--output text) && printf "$s" && echo; [[ -z "$s" ]] || [[ "$s" = "terminated" ]] || [[ "$s" = "shutting-down" ]]'
             ),
             # Test cancelling the spot cluster during spot job being setup.
-            f'sky spot launch --cloud aws --region {region} -n {name}-2 tests/test_yamls/long_setup.yaml  -y -d',
+            f'sky spot launch --cloud aws --region {region} -n {name}-2 tests/test_yamls/test_long_setup.yaml  -y -d',
             'sleep 300',
             f'sky spot cancel -y -n {name}-2',
             'sleep 5',
@@ -974,6 +1009,7 @@ def test_spot_storage():
         test = Test(
             'managed-spot-storage',
             [
+                *storage_setup_commands,
                 f'sky spot launch -n {name} {file_path} -y',
                 'sleep 60',  # Wait the spot queue to be updated
                 f'sky spot queue | grep {name} | grep SUCCEEDED',

tests/test_yamls/long_setup.yaml renamed to tests/test_yamls/test_long_setup.yaml

@@ -0,0 +1,42 @@
+file_mounts:
+  # Mounting public buckets
+  /mount_public_s3:
+    source: s3://digitalcorpora
+    mode: MOUNT
+
+  # Mounting public buckets
+  /mount_public_gcp:
+    source: gs://gcp-public-data-sentinel-2
+    mode: MOUNT
+
+  # Mounting private buckets in COPY mode
+  /mount_private_copy:
+    name: {{storage_name}}
+    source: ~/tmp-workdir
+    mode: COPY
+
+  # Mounting private buckets in MOUNT mode
+  /mount_private_mount:
+    name: {{storage_name}}
+    source: ~/tmp-workdir
+    mode: MOUNT
+
+run: |
+  set -ex
+  
+  # Check public bucket contents
+  ls -ltr /mount_public_s3/corpora
+  ls -ltr /mount_public_gcp/tiles
+  
+  # Check private bucket contents
+  ls -ltr /mount_private_copy/foo
+  ls -ltr /mount_private_copy/tmp\ file
+  ls -ltr /mount_private_mount/foo
+  ls -ltr /mount_private_mount/tmp\ file
+  
+  # Symlinks are not copied to buckets
+  ! ls /mount_private_copy/circle-link
+  ! ls /mount_private_mount/circle-link
+  
+  # Write to private bucket in MOUNT mode should pass
+  echo "hello" > /mount_private_mount/hello.txt