Add support for config prePackagedDownloadPath and EEGChunksPath

Author: laemtl

python/bids_import.py

@@ -38,7 +38,7 @@ def main():
 
     long_options = [
         "help",            "profile=",      "directory=",
-        "createcandidate", "createsession", "idsvalidation", "nobidsvalidation", "type", "verbose"
+        "createcandidate", "createsession", "idsvalidation", "nobidsvalidation", "type=", "verbose"
     ]
     usage        = (
         '\n'
@@ -58,7 +58,7 @@ def main():
     )
 
     try:
-        opts, args = getopt.getopt(sys.argv[1:], 'hp:d:csiv', long_options)
+        opts, args = getopt.getopt(sys.argv[1:], 'hp:d:csint:v', long_options)
     except getopt.GetoptError:
         print(usage)
         sys.exit(lib.exitcode.GETOPT_FAILURE)
@@ -88,15 +88,15 @@ def main():
     config_file = input_error_checking(profile, bids_dir, usage)
 
     dataset_json = bids_dir + "/dataset_description.json"
-    if not os.path.isfile(dataset_json):
+    if not os.path.isfile(dataset_json) and not type:
         print('No dataset_description.json found. Please run with the --type option.')
         print(usage)
-        sys.exit()
+        sys.exit(lib.exitcode.MISSING_ARG)
 
-    if type not in ('raw', 'derivatives'):
+    if type and type not in ('raw', 'derivatives'):
         print("--type must be one of 'raw', 'derivatives'")
         print(usage)
-        sys.exit()
+        sys.exit(lib.exitcode.MISSING_ARG)
 
     # database connection
     db = Database(config_file.mysql, verbose)

python/ingest_eeg_bids_datasets.py

@@ -150,23 +150,23 @@ def main():
 
       try:
         result = subprocess.run(command, shell = True, capture_output=True)
-        
+
         if result.stdout:
           print(result.stdout.decode('utf-8'))
 
-        if not result.stderr:
+        if result.stderr:
+          print(
+            f'ERROR: EEG Dataset with uploadID {uploadid} ingestion log:\n ' + result.stderr.decode('utf-8')
+          )
+
+        if result.returncode == 0:
           db.update(
             "UPDATE electrophysiology_uploader SET Status = 'Ingested' WHERE UploadID = %s",
             (uploadid,)
           )
           print('EEG Dataset with uploadID ' + uploadid + ' successfully ingested')
           continue
 
-        print(
-          f'ERROR: EEG Dataset with uploadID {uploadid} failed ingestion. \
-          Error was:\n ' + result.stderr.decode('utf-8')
-        )
-      
       except OSError:
         print('ERROR: ' + script + ' not found')
 

python/lib/eeg.py

@@ -14,7 +14,7 @@
 from lib.database_lib.physiologicalannotationfile    import PhysiologicalAnnotationFile
 from lib.database_lib.physiologicalannotationarchive import PhysiologicalAnnotationArchive
 from lib.database_lib.physiologicalannotationrel     import PhysiologicalAnnotationRel
-
+from lib.database_lib.config import Config
 
 __license__ = "GPLv3"
 
@@ -105,6 +105,9 @@ def __init__(self, bids_reader, bids_sub_id, bids_ses_id, bids_modality, db,
          :type type             : string
         """
 
+        # config
+        self.config_db_obj = Config(db, verbose)
+
         # load bids objects
         self.bids_reader   = bids_reader
         self.bids_layout   = bids_reader.bids_layout
@@ -948,9 +951,7 @@ def create_and_insert_archive(self, files_to_archive, archive_rel_name,
             else:
                 return
 
-        # create the archive file
-        archive_rel_name = archive_rel_name.replace(self.data_dir, '')
-        utilities.create_archive(files_to_archive, archive_rel_name, self.data_dir)
+        (archive_rel_name, archive_full_path) = self.create_archive(files_to_archive, archive_rel_name)
 
         # insert the archive file in physiological_archive
         blake2 = utilities.compute_blake2b_hash(archive_full_path)
@@ -1003,9 +1004,22 @@ def create_and_insert_annotation_archive(self, files_to_archive, archive_rel_nam
             else:
                 return
 
-        # create the archive file
-        utilities.create_archive(files_to_archive, archive_rel_name, self.data_dir)
+        (archive_rel_name, archive_full_path) = self.create_archive(files_to_archive, archive_rel_name)
 
         # insert the archive into the physiological_annotation_archive table
         blake2 = utilities.compute_blake2b_hash(archive_full_path)
         physiological_annotation_archive_obj.insert(eeg_file_id, blake2, archive_rel_name)
+
+    def create_archive(self, files_to_archive, archive_rel_name):
+        # create the archive file
+        package_path = self.config_db_obj.get_config("prePackagedDownloadPath")
+        if package_path:
+            raw_package_dir = os.path.join(package_path, 'raw')
+            os.makedirs(raw_package_dir, exist_ok=True)
+            archive_rel_name = os.path.basename(archive_rel_name)
+            archive_full_path = os.path.join(raw_package_dir, archive_rel_name)
+            utilities.create_archive(files_to_archive, archive_full_path)
+        else:
+            archive_full_path = os.path.join(self.data_dir, archive_rel_name)
+            utilities.create_archive(files_to_archive, archive_full_path)
+        return (archive_rel_name, archive_full_path)

python/lib/physiological.py

@@ -9,7 +9,7 @@
 from lib.database_lib.physiologicalannotationparameter import PhysiologicalAnnotationParameter
 from lib.database_lib.physiologicalannotationlabel import PhysiologicalAnnotationLabel
 from lib.database_lib.physiologicalannotationinstance import PhysiologicalAnnotationInstance
-
+from lib.database_lib.config import Config
 
 __license__ = "GPLv3"
 
@@ -56,6 +56,7 @@ def __init__(self, db, verbose):
 
         self.db      = db
         self.verbose = verbose
+        self.config_db_obj = Config(self.db, self.verbose)
 
         self.physiological_annotation_file_obj      = PhysiologicalAnnotationFile(self.db, self.verbose)
         self.physiological_annotation_parameter_obj = PhysiologicalAnnotationParameter(self.db, self.verbose)
@@ -179,6 +180,7 @@ def insert_physio_parameter_file(self, physiological_file_id,
         parameter_file_values = (
             physiological_file_id, parameter_type_id, value
         )
+
         self.db.insert(
             table_name='physiological_parameter_file',
             column_names=parameter_file_fields,
@@ -977,32 +979,32 @@ def create_chunks_for_visualization(self, physio_file_id, data_dir):
 
         # check if chunks already exists for this PhysiologicalFileID
         results    = self.grep_parameter_value_from_file_id(
-            physio_file_id, 'electrophyiology_chunked_dataset_path'
+            physio_file_id, 'electrophysiology_chunked_dataset_path'
         )
         chunk_path = results['Value'] if results else None
 
-        # determine which script to run based on the file type
-        command   = None
-        script    = None
+        # No chunks found
         if not chunk_path:
-            file_type    = self.grep_file_type_from_file_id(physio_file_id)
-            file_path    = self.grep_file_path_from_file_id(physio_file_id)
-            # the bids_rel_dir is the first two directories in file_path (
-            # bids_imports/BIDS_dataset_name_BIDSVersion)
-            bids_rel_dir   = file_path.split('/')[0] + '/' + file_path.split('/')[1]
-            chunk_root_dir = data_dir + bids_rel_dir + '_chunks' + '/'
-            # the final chunk path will be /data/%PROJECT%/data/bids_imports
-            # /BIDS_dataset_name_BIDSVersion_chunks/EEG_FILENAME.chunks
-            chunk_path = chunk_root_dir + os.path.splitext(os.path.basename(file_path))[0] + '.chunks'
+            script    = None
+            file_path = self.grep_file_path_from_file_id(physio_file_id)
+
+            chunk_root_dir = self.config_db_obj.get_config("EEGChunksPath")
+            if not chunk_root_dir:
+                # the bids_rel_dir is the first two directories in file_path (
+                # bids_imports/BIDS_dataset_name_BIDSVersion)
+                bids_rel_dir   = file_path.split('/')[0] + '/' + file_path.split('/')[1]
+                chunk_root_dir = data_dir + bids_rel_dir + '_chunks' + '/'
+
+            # determine which script to run based on the file type
+            file_type = self.grep_file_type_from_file_id(physio_file_id)
             if file_type == 'set':
                 script = os.environ['LORIS_MRI'] + '/python/react-series-data-viewer/eeglab_to_chunks.py'
                 command = 'python ' + script + ' ' + data_dir + file_path + ' --destination ' + chunk_root_dir
             elif file_type == 'edf':
                 script = os.environ['LORIS_MRI'] + '/python/react-series-data-viewer/edf_to_chunks.py'
                 command = 'python ' + script + ' ' + data_dir + file_path + ' --destination ' + chunk_root_dir
 
-        # chunk the electrophysiology dataset if a command was determined above
-        if command:
+            # chunk the electrophysiology dataset if a command was determined above
             try:
                 subprocess.call(
                     command,
@@ -1016,9 +1018,12 @@ def create_chunks_for_visualization(self, physio_file_id, data_dir):
                 print('ERROR: ' + script + ' not found')
                 sys.exit(lib.exitcode.CHUNK_CREATION_FAILURE)
 
+            # the final chunk path will be /data/%PROJECT%/data/bids_imports
+            # /BIDS_dataset_name_BIDSVersion_chunks/EEG_FILENAME.chunks
+            chunk_path = chunk_root_dir + os.path.splitext(os.path.basename(file_path))[0] + '.chunks'
             if os.path.isdir(chunk_path):
                 self.insert_physio_parameter_file(
                     physiological_file_id = physio_file_id,
-                    parameter_name = 'electrophyiology_chunked_dataset_path',
+                    parameter_name = 'electrophysiology_chunked_dataset_path',
                     value = chunk_path.replace(data_dir, '')
                 )

python/lib/utilities.py

@@ -138,7 +138,7 @@ def create_dir(dir_name, verbose):
     return dir_name
 
 
-def create_archive(files_to_archive, archive_rel_name, data_dir):
+def create_archive(files_to_archive, archive_path):
     """
     Creates an archive with the files listed in the files_to_archive tuple.
 
@@ -151,8 +151,8 @@ def create_archive(files_to_archive, archive_rel_name, data_dir):
     """
 
     # if the archive does not already exists, create it
-    if not os.path.isfile(data_dir + archive_rel_name):
-        tar = tarfile.open(data_dir + archive_rel_name, "w:gz")
+    if not os.path.isfile(archive_path):
+        tar = tarfile.open(archive_path, "w:gz")
         for file in files_to_archive:
             filename = os.path.basename(file)
             tar.add(file, arcname=filename, recursive=False)

python/react-series-data-viewer/eeglab_to_chunks.py

@@ -30,7 +30,7 @@ def load_channels(path):
     args = parser.parse_args()
     for path in args.files:
         eeg = mne_eeglab._check_load_mat(path, None)
-        eeglab_info = mne_eeglab._get_info(eeg, eog=(), montage_units="auto")
+        eeglab_info = mne_eeglab._get_info(eeg, eog=())
         channel_names = eeglab_info[0]['ch_names']
 
         if args.channel_index < 0: