Support --outdirname for datasets fixes #63 (#75)

sahilavaran · web-flow · commit d28df7ea8803 · 2024-12-23T15:07:11.000Z
* support --outdirname for ml models, partially fixed #63 * support --outdirname for ml datasets fixes #63
diff --git a/script/get-dataset-imagenet-aux/_cm.yaml b/script/get-dataset-imagenet-aux/_cm.yaml
@@ -14,6 +14,8 @@ prehook_deps:
   extra_cache_tags: imagenet-aux,dataset-aux
   force_cache: true
   tags: download-and-extract,_extract,_wget
+  force_env_keys:
+    - CM_OUTDIRNAME  
   update_tags_from_env_with_prefix:
     _url.:
     - CM_PACKAGE_URL
diff --git a/script/get-dataset-imagenet-calibration/_cm.yaml b/script/get-dataset-imagenet-calibration/_cm.yaml
@@ -10,6 +10,8 @@ category: "AI/ML datasets"
 
 deps:
   - tags: download,file
+    force_env_keys:
+      - CM_OUTDIRNAME
     force_cache: true
     extra_cache_tags: imagenet-calibration,imagenet,calibration
     names:
diff --git a/script/get-dataset-imagenet-val/_cm.yaml b/script/get-dataset-imagenet-val/_cm.yaml
@@ -35,6 +35,8 @@ prehook_deps:
   env:
     CM_EXTRACT_TO_FOLDER: imagenet-2012-val
   tags: download-and-extract,file,_extract
+  force_env_keys:
+    - CM_OUTDIRNAME  
   update_tags_from_env:
   - CM_DAE_EXTRA_TAGS
   update_tags_from_env_with_prefix:
diff --git a/script/get-dataset-mlperf-inference-mixtral/_cm.yaml b/script/get-dataset-mlperf-inference-mixtral/_cm.yaml
@@ -8,9 +8,11 @@ new_env_keys:
 prehook_deps:
 - env:
     CM_DOWNLOAD_FINAL_ENV_NAME: CM_DATASET_PREPROCESSED_PATH
-  extra_cache_tags: mixtral,get-mixtral-dataset
+  extra_cache_tags: mixtral,get-mixtral-dataset  
   force_cache: true
   tags: download-and-extract
+  force_env_keys:
+    - CM_OUTDIRNAME  
   update_tags_from_env_with_prefix:
     _url.:
     - CM_PACKAGE_URL
diff --git a/script/get-dataset-openimages-annotations/_cm.yaml b/script/get-dataset-openimages-annotations/_cm.yaml
@@ -12,6 +12,8 @@ prehook_deps:
   extra_cache_tags: retinanet,get,dataset-openimages-annotations
   force_cache: true
   tags: download-and-extract,_wget,_extract
+  force_env_keys:
+    - CM_OUTDIRNAME  
   update_tags_from_env_with_prefix:
     _url.:
     - CM_PACKAGE_URL
diff --git a/script/get-dataset-openimages-calibration/_cm.yaml b/script/get-dataset-openimages-calibration/_cm.yaml
@@ -10,6 +10,8 @@ category: "AI/ML datasets"
 
 deps:
   - tags: download,file
+    force_env_keys:
+      - CM_OUTDIRNAME
     force_cache: true
     extra_cache_tags: openimages-calibration,openimages,calibration
     names:
diff --git a/script/get-dataset-openorca/_cm.yaml b/script/get-dataset-openorca/_cm.yaml
@@ -15,6 +15,8 @@ deps:
   names:
   - openorca-src
   tags: get,git,repo,_lfs,_repo.https://huggingface.co/datasets/Open-Orca/OpenOrca
+  force_env_keys:
+    - CM_OUTDIRNAME  
 env:
   CM_DATASET: OPENORCA
 new_env_keys:
diff --git a/script/get-dataset-squad-vocab/_cm.yaml b/script/get-dataset-squad-vocab/_cm.yaml
@@ -12,6 +12,8 @@ prehook_deps:
   extra_cache_tags: bert,get,dataset-squad-vocab
   force_cache: true
   tags: download-and-extract,_wget
+  force_env_keys:
+    - CM_OUTDIRNAME  
   update_tags_from_env_with_prefix:
     _url.:
     - CM_PACKAGE_URL
diff --git a/script/get-dataset-squad/_cm.yaml b/script/get-dataset-squad/_cm.yaml
@@ -16,6 +16,8 @@ prehook_deps:
   extra_cache_tags: bert,get,dataset-squad
   force_cache: true
   tags: download-and-extract,_wget
+  force_env_keys:
+    - CM_OUTDIRNAME  
   update_tags_from_env_with_prefix:
     _url.:
     - CM_PACKAGE_URL
