Exclude .git for cloud storage (skypilot-org#1494)

* s3 exclude .git

* gcs

* comments

* warning and fix doc

* Update sky/data/storage.py

Co-authored-by: Zongheng Yang <[email protected]>

* fix

Co-authored-by: Zongheng Yang <[email protected]>

Author: infwinston

docs/source/examples/spot-jobs.rst

@@ -42,7 +42,7 @@ We can launch it with the following:
 
   # Assume your working directory is under `~/transformers`.
   # To make this example work, please run the following command:
-  # git clone https://github.com/huggingface/transformers.git ~/transformers
+  # git clone https://github.com/huggingface/transformers.git ~/transformers -b v4.18.0
   workdir: ~/transformers
 
   setup: |
@@ -51,7 +51,6 @@ We can launch it with the following:
     # to pass the key in the command line, during `sky spot launch`.
     echo export WANDB_API_KEY=[YOUR-WANDB-API-KEY] >> ~/.bashrc
 
-    git checkout v4.18.0
     pip install -e .
     cd examples/pytorch/question-answering/
     pip install -r requirements.txt torch==1.12.1+cu113 --extra-index-url https://download.pytorch.org/whl/cu113
@@ -117,7 +116,7 @@ Below we show an `example <https://github.com/skypilot-org/skypilot/blob/master/
 
   # Assume your working directory is under `~/transformers`.
   # To make this example work, please run the following command:
-  # git clone https://github.com/huggingface/transformers.git ~/transformers
+  # git clone https://github.com/huggingface/transformers.git ~/transformers -b v4.18.0
   workdir: ~/transformers
 
   file_mounts:
@@ -131,7 +130,6 @@ Below we show an `example <https://github.com/skypilot-org/skypilot/blob/master/
     # to pass the key in the command line, during `sky spot launch`.
     echo export WANDB_API_KEY=[YOUR-WANDB-API-KEY] >> ~/.bashrc
 
-    git checkout v4.18.0
     pip install -e .
     cd examples/pytorch/question-answering/
     pip install -r requirements.txt

examples/spot/bert_qa.yaml

@@ -5,7 +5,7 @@ resources:
 
 # Assume your working directory is under `~/transformers`.
 # To make this example work, please run the following command:
-# git clone https://github.com/huggingface/transformers.git ~/transformers
+# git clone https://github.com/huggingface/transformers.git ~/transformers -b v4.18.0
 workdir: ~/transformers
 
 file_mounts:
@@ -17,7 +17,6 @@ setup: |
     # Fill in your wandb key: copy from https://wandb.ai/authorize
     echo export WANDB_API_KEY=[YOUR-WANDB-API-KEY] >> ~/.bashrc
 
-    git checkout v4.18.0
     pip install -e .
     cd examples/pytorch/question-answering/
     pip install -r requirements.txt torch==1.12.1+cu113 --extra-index-url https://download.pytorch.org/whl/cu113

sky/data/storage.py

@@ -721,6 +721,10 @@ def sync_all_stores(self):
     def _sync_store(self, store: AbstractStore):
         """Runs the upload routine for the store and handles failures"""
         try:
+            if self.source is not None and os.path.isdir(
+                    os.path.join(self.source, '.git')):
+                logger.warning(f'\'.git\' directory under \'{self.source}\' '
+                               'is excluded during sync.')
             store.upload()
         except exceptions.StorageUploadError:
             logger.error(f'Could not upload {self.source} to store '
@@ -906,9 +910,11 @@ def get_file_sync_command(base_dir_path, file_names):
             return sync_command
 
         def get_dir_sync_command(src_dir_path, dest_dir_name):
-            sync_command = ('aws s3 sync --no-follow-symlinks '
-                            f'{src_dir_path} '
-                            f's3://{self.name}/{dest_dir_name}')
+            # we exclude .git directory from the sync
+            sync_command = (
+                'aws s3 sync --no-follow-symlinks --exclude ".git/*" '
+                f'{src_dir_path} '
+                f's3://{self.name}/{dest_dir_name}')
             return sync_command
 
         # Generate message for upload
@@ -1219,7 +1225,8 @@ def get_file_sync_command(base_dir_path, file_names):
             return sync_command
 
         def get_dir_sync_command(src_dir_path, dest_dir_name):
-            sync_command = (f'gsutil -m rsync -r {src_dir_path} '
+            # we exclude .git directory from the sync
+            sync_command = (f'gsutil -m rsync -r -x \'.git/*\' {src_dir_path} '
                             f'gs://{self.name}/{dest_dir_name}')
             return sync_command