Skip download if the asset is already there

Author: Kerollmops

download.sh

@@ -41,30 +41,46 @@ cargo build --release -p parquet2mat
 max_parallel=10
 
 # Call the function to download parquet files in parallel
-cat dbpedia-1536.urls | xargs -n 1 -P $max_parallel -I {} bash -c 'download_file {}'
-cargo run --release -p parquet2mat -- \
-    tmp-assets/* \
-    --embedding-name 'text-embedding-3-large-1536-embedding' \
-    --output assets/db-pedia-OpenAI-text-embedding-ada-002.mat
-rm -rf tmp-assets/*
+output=assets/db-pedia-OpenAI-text-embedding-ada-002.mat
+if [ ! -f $output ]; then
+    cat dbpedia-1536.urls | xargs -n 1 -P $max_parallel -I {} bash -c 'download_file {}'
+    cargo run --release -p parquet2mat -- \
+        tmp-assets/* \
+        --embedding-name 'text-embedding-3-large-1536-embedding' \
+        --output $output
+    rm -rf tmp-assets/*
+fi
 
 # Call the function to download parquet files in parallel
-cat dbpedia-3072.urls | xargs -n 1 -P $max_parallel -I {} bash -c 'download_file {}'
-cargo run --release -p parquet2mat -- \
-    tmp-assets/* \
-    --embedding-name 'text-embedding-3-large-3072-embedding' \
-    --output assets/db-pedia-OpenAI-text-embedding-3-large.mat
-rm -rf tmp-assets/*
+output=assets/db-pedia-OpenAI-text-embedding-3-large.mat
+if [ ! -f $output ]; then
+    cat dbpedia-3072.urls | xargs -n 1 -P $max_parallel -I {} bash -c 'download_file {}'
+    cargo run --release -p parquet2mat -- \
+        tmp-assets/* \
+        --embedding-name 'text-embedding-3-large-3072-embedding' \
+        --output $output
+    rm -rf tmp-assets/*
+fi
 
 # Call the function to download parquet files in parallel
-cat wikipedia-768.urls | xargs -n 1 -P $max_parallel -I {} bash -c 'download_file {}'
-cargo run --release -p parquet2mat -- \
-    tmp-assets/* \
-    --embedding-name 'emb' \
-    --output assets/wikipedia-22-12-simple-embeddings.mat
-rm -rf tmp-assets/*
+output=assets/wikipedia-22-12-simple-embeddings.mat
+if [ ! -f $output ]; then
+    cat wikipedia-768.urls | xargs -n 1 -P $max_parallel -I {} bash -c 'download_file {}'
+    cargo run --release -p parquet2mat -- \
+        tmp-assets/* \
+        --embedding-name 'emb' \
+        --output $output
+    rm -rf tmp-assets/*
+fi
 
-curl -o assets/hn-posts.mat 'https://static.wilsonl.in/hackerverse/dataset/post-embs-data.mat'
-curl -o assets/hn-top-posts.mat 'https://static.wilsonl.in/hackerverse/dataset/toppost-embs-data.mat'
+output=assets/hn-posts.mat
+if [ ! -f $output ]; then
+    curl -o $output 'https://static.wilsonl.in/hackerverse/dataset/post-embs-data.mat'
+fi
+
+output=assets/hn-top-posts.mat
+if [ ! -f $output ]; then
+    curl -o $output 'https://static.wilsonl.in/hackerverse/dataset/toppost-embs-data.mat'
+fi
 
 rm -rf tmp-assets