Integration test for run_dicom_archive_loader.py

.github/workflows/integration.yml

@@ -11,6 +11,14 @@ env:
   DATABASE_NAME: TestDatabase
   DATABASE_USERNAME: TestUsername
   DATABASE_PASSWORD: TestPassword
+  BUCKET_URL: https://ace-minio-1.loris.ca:9000
+  BUCKET_NAME: loris-rb-data
+  # NOTE: These are read-only keys on public data.
+  # Ideally, we would like to hide these keys. However, both GitHub variables and GitHub secrets
+  # are not accessible from pull requests from forks. Since we want to run integration tests on
+  # pull requests, we define these variables here.
+  BUCKET_ACCESS_KEY: lorisadmin-ro
+  BUCKET_SECRET_KEY: Tn=qP3LupmXnMuc
 
 jobs:
   docker:
@@ -55,7 +63,20 @@ jobs:
         tags: loris-mri
         load: true
         cache-from: type=gha,scope=loris-mri
-        cache-to: type=gha,scope=loris-mri
+        cache-to: type=gha,mode=max,scope=loris-mri
+
+    # NOTE: Ideally, we would like to mount the S3 bucket in the Docker image, but since it
+    # interacts with the kernel to add a file system, it is hard to do so.
+    - name: Mount imaging files S3 bucket
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y s3fs fuse kmod
+        sudo modprobe fuse
+        sudo mkdir /data-imaging
+        touch .passwd-s3fs
+        chmod 600 .passwd-s3fs
+        echo ${{ env.BUCKET_ACCESS_KEY }}:${{ env.BUCKET_SECRET_KEY }} > .passwd-s3fs
+        sudo s3fs ${{ env.BUCKET_NAME }} /data-imaging -o url=${{ env.BUCKET_URL }} -o passwd_file=.passwd-s3fs -o use_path_request_style -o allow_other
 
     - name: Run integration tests
       run: docker compose --file ./test/docker-compose.yml run mri pytest python/tests/integration

dicom-archive/database_config_template.py

@@ -1,10 +1,10 @@
 #!/usr/bin/env python
 
 import re
+
+from lib.config_file import CreateVisitInfo, DatabaseConfig, SubjectInfo
 from lib.database import Database
 from lib.imaging import Imaging
-from lib.config_file import CreateVisitInfo, DatabaseConfig, S3Config, SubjectInfo
-
 
 mysql: DatabaseConfig = DatabaseConfig(
     host     = 'DBHOST',
@@ -14,13 +14,13 @@
     port     = 3306,
 )
 
-# This statement can be omitted if the project does not use AWS S3.
-s3: S3Config = S3Config(
-    aws_access_key_id     = 'AWS_ACCESS_KEY_ID',
-    aws_secret_access_key = 'AWS_SECRET_ACCESS_KEY',
-    aws_s3_endpoint_url   = 'AWS_S3_ENDPOINT',
-    aws_s3_bucket_name    = 'AWS_S3_BUCKET_NAME',
-)
+# Uncomment this statement if your project uses AWS S3.
+# s3: S3Config = S3Config(
+#     aws_access_key_id     = 'AWS_ACCESS_KEY_ID',
+#     aws_secret_access_key = 'AWS_SECRET_ACCESS_KEY',
+#     aws_s3_endpoint_url   = 'AWS_S3_ENDPOINT',
+#     aws_s3_bucket_name    = 'AWS_S3_BUCKET_NAME',
+# )
 
 
 def get_subject_info(db: Database, subject_name: str, scanner_id: int | None = None) -> SubjectInfo | None:

install/install_database.sql

@@ -13,11 +13,11 @@ UPDATE Config SET Value = @project
   WHERE ConfigID = (SELECT ID FROM ConfigSettings WHERE Name = 'prefix');
 UPDATE Config SET Value = @minc_dir
   WHERE ConfigID = (SELECT ID FROM ConfigSettings WHERE Name = 'MINCToolsPath');
-UPDATE Config SET Value = CONCAT('/data/', @project, '/data/')
+UPDATE Config SET Value = CONCAT('/data/', @project, '/')
   WHERE ConfigID = (SELECT ID FROM ConfigSettings WHERE Name = 'dataDirBasepath');
-UPDATE Config SET Value = CONCAT('/data/', @project, '/data/')
+UPDATE Config SET Value = CONCAT('/data/', @project, '/')
   WHERE ConfigID = (SELECT ID FROM ConfigSettings WHERE Name = 'imagePath');
-UPDATE Config SET Value = CONCAT('/data/', @project, '/data/tarchive/')
+UPDATE Config SET Value = CONCAT('/data/', @project, '/tarchive/')
   WHERE ConfigID = (SELECT ID FROM ConfigSettings WHERE Name = 'tarchiveLibraryDir');
 UPDATE Config SET Value = CONCAT('/opt/', @project, '/bin/mri/dicom-archive/get_dicom_info.pl')
   WHERE ConfigID = (SELECT ID FROM ConfigSettings WHERE Name = 'get_dicom_info');

python/lib/database.py

@@ -108,7 +108,8 @@ def connect(self):
                 user=self.user_name,
                 passwd=self.password,
                 port=self.port,
-                db=self.db_name
+                db=self.db_name,
+                autocommit=True,
             )
             # self.cnx.cursor = self.cnx.cursor(prepared=True)
         except MySQLdb.Error as err:
@@ -178,7 +179,6 @@ def insert(self, table_name, column_names, values, get_last_id=False):
             else:
                 # else, values is a tuple and want to execute only one insert
                 cursor.execute(query, values)
-            self.con.commit()
             last_id = cursor.lastrowid
             cursor.close()
         except MySQLdb.Error as err:
@@ -206,7 +206,6 @@ def update(self, query, args):
         try:
             cursor = self.con.cursor()
             cursor.execute(query, args)
-            self.con.commit()
         except MySQLdb.Error as err:
             raise Exception("Update query failure: " + format(err))
 

python/lib/db/model/file.py

@@ -1,16 +1,18 @@
 from datetime import date
 from typing import Optional
 
-from sqlalchemy.orm import Mapped, mapped_column
+from sqlalchemy import ForeignKey
+from sqlalchemy.orm import Mapped, mapped_column, relationship
 
+import lib.db.model.session as db_session
 from lib.db.base import Base
 
 
 class DbFile(Base):
     __tablename__ = 'files'
 
     id                             : Mapped[int]             = mapped_column('FileID', primary_key=True)
-    session_id                     : Mapped[int]             = mapped_column('SessionID')
+    session_id                     : Mapped[int]             = mapped_column('SessionID', ForeignKey('session.ID'))
     file_name                      : Mapped[str]             = mapped_column('File')
     series_uid                     : Mapped[Optional[str]]   = mapped_column('SeriesUID')
     echo_time                      : Mapped[Optional[float]] = mapped_column('EchoTime')
@@ -32,3 +34,5 @@ class DbFile(Base):
     scanner_id                     : Mapped[Optional[int]]   = mapped_column('ScannerID')
     acquisition_order_per_modality : Mapped[Optional[int]]   = mapped_column('AcqOrderPerModality')
     acquisition_date               : Mapped[Optional[date]]  = mapped_column('AcquisitionDate')
+
+    session : Mapped['db_session.DbSession'] = relationship('DbSession', back_populates='files')

python/lib/db/model/session.py

@@ -5,6 +5,7 @@
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 import lib.db.model.candidate as db_candidate
+import lib.db.model.file as db_file
 import lib.db.model.project as db_project
 import lib.db.model.site as db_site
 from lib.db.base import Base
@@ -51,5 +52,6 @@ class DbSession(Base):
     language_id              : Mapped[Optional[int]]      = mapped_column('languageID')
 
     candidate : Mapped['db_candidate.DbCandidate'] = relationship('DbCandidate', back_populates='sessions')
+    files     : Mapped[list['db_file.DbFile']]     = relationship('DbFile', back_populates='session')
     project   : Mapped['db_project.DbProject']     = relationship('DbProject')
     site      : Mapped['db_site.DbSite']           = relationship('DbSite')

python/lib/db/query/config.py

@@ -1,4 +1,4 @@
-from sqlalchemy import select
+from sqlalchemy import select, update
 from sqlalchemy.orm import Session as Database
 
 from lib.db.model.config import DbConfig
@@ -15,3 +15,19 @@ def get_config_with_setting_name(db: Database, name: str):
         .join(DbConfig.setting)
         .where(DbConfigSetting.name == name)
     ).scalar_one()
+
+
+def set_config_with_setting_name(db: Database, name: str, value: str):
+    """
+    Set a single configuration entry from the database using its configuration setting name, or
+    raise an exception if the configuration setting is not found.
+    """
+
+    config_setting = db.execute(select(DbConfigSetting)
+        .where(DbConfigSetting.name == name)
+    ).scalar_one()
+
+    db.execute(update(DbConfig)
+        .where(DbConfig.setting == config_setting)
+        .values(value = value)
+    )

python/lib/db/query/mri_upload.py

@@ -12,3 +12,14 @@ def try_get_mri_upload_with_id(db: Database, id: int):
     return db.execute(select(DbMriUpload)
         .where(DbMriUpload.id == id)
     ).scalar_one_or_none()
+
+
+def get_mri_upload_with_patient_name(db: Database, patient_name: str):
+    """
+    Get an MRI upload from the database using its patient name, or throw an exception if no MRI
+    upload is found.
+    """
+
+    return db.execute(select(DbMriUpload)
+        .where(DbMriUpload.patient_name == patient_name)
+    ).scalar_one()

python/tests/integration/scripts/test_run_dicom_archive_loader.py

@@ -0,0 +1,46 @@
+import subprocess
+
+from lib.db.query.config import set_config_with_setting_name
+from lib.db.query.mri_upload import get_mri_upload_with_patient_name
+from tests.util.database import get_integration_database_session
+from tests.util.file_system import check_file_tree
+
+
+def test():
+    db = get_integration_database_session()
+
+    # Set the configuration to use the DICOM to BIDS pipeline
+    set_config_with_setting_name(db, 'converter', 'dcm2niix')
+    db.commit()
+
+    # Run the script to test
+    process = subprocess.run([
+        'run_dicom_archive_loader.py',
+        '--profile', 'database_config.py',
+        '--tarchive_path', '/data/loris/tarchive/DCM_2015-07-07_ImagingUpload-14-30-FoTt1K.tar',
+    ], capture_output=True)
+
+    # Print the standard output and error for debugging
+    print(f'STDOUT:\n{process.stdout.decode()}')
+    print(f'STDERR:\n{process.stderr.decode()}')
+
+    # Check that the return code and standard error are correct
+    assert process.returncode == 0
+    assert process.stderr == b''
+
+    # Check that the expected files have been created
+    assert check_file_tree('/data/loris/assembly_bids', {
+        'sub-300001': {
+            'ses-V2': {
+                'anat': {
+                    'sub-300001_ses-V2_run-1_T1w.json': None,
+                    'sub-300001_ses-V2_run-1_T1w.nii.gz': None,
+                }
+            }
+        }
+    })
+
+    # Check that the expected data has been inserted in the database
+    mri_upload = get_mri_upload_with_patient_name(db, 'MTL001_300001_V2')
+    assert mri_upload.session is not None
+    assert len(mri_upload.session.files) == 1

python/tests/util/database.py

@@ -28,3 +28,12 @@ def get_integration_database_engine():
     sys.path.append(os.path.dirname(config_file))
     config = __import__(os.path.basename(config_file[:-3]))
     return get_database_engine(config.mysql)
+
+
+def get_integration_database_session():
+    """
+    Get an SQLAlchemy session for the integration testing database using the configuration from the
+    Python configuration file.
+    """
+
+    return Session(get_integration_database_engine())

python/tests/util/file_system.py

@@ -0,0 +1,28 @@
+import os
+
+FileTree = dict[str, 'FileTree'] | None
+"""
+Type that represents a file hierarchy relative to a path.
+- `None` means that the path refers to a file.
+- `dict[str, FileTree]` means that the path refers to a directory, with the entries of the
+  dictionary as sub-trees.
+"""
+
+
+def check_file_tree(path: str, file_tree: FileTree):
+    """
+    Check that a path has at least all the directories and files of a file tree.
+    """
+
+    if file_tree is None:
+        return os.path.isfile(path)
+
+    if not os.path.isdir(path):
+        return False
+
+    for sub_dir_name, sub_file_tree in file_tree.items():
+        sub_dir_path = os.path.join(path, sub_dir_name)
+        if not check_file_tree(sub_dir_path, sub_file_tree):
+            return False
+
+    return True

test/db.Dockerfile

@@ -30,7 +30,7 @@ RUN ( \
 # Copy the LORIS-MRI database installation script and add it to the compiled SQL file.
 COPY install/install_database.sql /tmp/install_database.sql
 RUN echo "SET @email := 'root@localhost'; SET @project := 'loris'; SET @minc_dir = '/opt/minc/1.9.18';" >> source.sql
-RUN cat /tmp/install_database.sql >> /docker-entrypoint-initdb.d/source.sql
+RUN cat /tmp/install_database.sql >> source.sql
 
 # By default, MariaDB runs the SQL files provided by the user at the time of the first startup of
 # the image. However, we want to populate the database at build time, we therefore need to manually

test/docker-compose.yml

@@ -11,6 +11,8 @@ services:
       retries: 5
   mri:
     image: loris-mri
+    volumes:
+      - /data-imaging:/data-imaging
     depends_on:
       db:
         condition: service_healthy

test/entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Create a writable directory with links to the imaging dataset files
+replicate_raisinbread_for_mcin_dev_vm.pl /data-imaging /data/loris
+
+# Run the provided command (usually the integration test command)
+exec "$@"

test/imaging_install_test.sh

@@ -14,16 +14,16 @@ mridir="/opt/loris/bin/mri"
 #############################Create directories########################################
 #######################################################################################
 echo "Creating the data directories"
-  sudo -S su $USER -c "mkdir -m 2770 -p /data/$PROJ/data/"
-  sudo -S su $USER -c "mkdir -m 770 -p /data/$PROJ/data/trashbin"         #holds mincs that didn't match protocol
-  sudo -S su $USER -c "mkdir -m 770 -p /data/$PROJ/data/tarchive"         #holds tared dicom-folder
-  sudo -S su $USER -c "mkdir -m 770 -p /data/$PROJ/data/hrrtarchive"      #holds tared hrrt-folder
-  sudo -S su $USER -c "mkdir -m 770 -p /data/$PROJ/data/pic"              #holds jpegs generated for the MRI-browser
-  sudo -S su $USER -c "mkdir -m 770 -p /data/$PROJ/data/logs"             #holds logs from pipeline script
-  sudo -S su $USER -c "mkdir -m 770 -p /data/$PROJ/data/assembly"         #holds the MINC files
-  sudo -S su $USER -c "mkdir -m 770 -p /data/$PROJ/data/assembly_bids"    #holds the BIDS files derived from DICOMs
-  sudo -S su $USER -c "mkdir -m 770 -p /data/$PROJ/data/batch_output"     #contains the result of the SGE (queue)
-  sudo -S su $USER -c "mkdir -m 770 -p /data/$PROJ/data/bids_imports"     #contains imported BIDS studies
+  sudo -S su $USER -c "mkdir -m 2770 -p /data/$PROJ/"
+  sudo -S su $USER -c "mkdir -m 770 -p /data/$PROJ/trashbin"         #holds mincs that didn't match protocol
+  sudo -S su $USER -c "mkdir -m 770 -p /data/$PROJ/tarchive"         #holds tared dicom-folder
+  sudo -S su $USER -c "mkdir -m 770 -p /data/$PROJ/hrrtarchive"      #holds tared hrrt-folder
+  sudo -S su $USER -c "mkdir -m 770 -p /data/$PROJ/pic"              #holds jpegs generated for the MRI-browser
+  sudo -S su $USER -c "mkdir -m 770 -p /data/$PROJ/logs"             #holds logs from pipeline script
+  sudo -S su $USER -c "mkdir -m 770 -p /data/$PROJ/assembly"         #holds the MINC files
+  sudo -S su $USER -c "mkdir -m 770 -p /data/$PROJ/assembly_bids"    #holds the BIDS files derived from DICOMs
+  sudo -S su $USER -c "mkdir -m 770 -p /data/$PROJ/batch_output"     #contains the result of the SGE (queue)
+  sudo -S su $USER -c "mkdir -m 770 -p /data/$PROJ/bids_imports"     #contains imported BIDS studies
   sudo -S su $USER -c "mkdir -m 770 -p $mridir/dicom-archive/.loris_mri"
 echo
 
@@ -71,8 +71,8 @@ sudo usermod -a -G $group $USER
 sudo chgrp $group -R /opt/$PROJ/
 sudo chgrp $group -R /data/$PROJ/
 
-#Setting group ID for all files/dirs under /data/$PROJ/data
-sudo chmod -R g+s /data/$PROJ/data/
+#Setting group ID for all files/dirs under /data/$PROJ/
+sudo chmod -R g+s /data/$PROJ/
 
 # Setting group permissions and group ID for all files/dirs under /data/incoming
 # If the directory was not created earlier, then instructions to do so manually are provided.

test/mri.Dockerfile

@@ -7,21 +7,13 @@
 # Install utilities #
 #####################
 
-# Update the package list and install build-essential, checkinstall, and cmake
-# Install some general dependencies
-RUN apt-get install -y build-essential checkinstall cmake libzip-dev mariadb-client
-
-# Install Perl and update CPAN
-RUN apt-get install -y perl && \
-    cpan CPAN
+# Install the dependencies of LORIS-MRI
+RUN apt-get install -y build-essential checkinstall cmake dcmtk dcm2niix libzip-dev mariadb-client perl
 
 # Install utilities
-# - `wget` is used by some installation commands
 # - `sudo` is used by the imaging install script
-RUN apt-get install -y wget sudo
-
-# Install the DICOM Toolkit
-RUN apt-get install -y dcmtk
+# - `wget` is used by some installation commands
+RUN apt-get install -y sudo wget
 
 ########################
 # Install MINC Toolkit #
@@ -43,13 +35,13 @@
 ENV MINC_TOOLKIT=/opt/minc/1.9.18
 ENV MINC_TOOLKIT_VERSION="1.9.18-20200813"
 ENV PATH=${MINC_TOOLKIT}/bin:${MINC_TOOLKIT}/pipeline:${PATH}
 ENV PERL5LIB=${MINC_TOOLKIT}/perl:${MINC_TOOLKIT}/pipeline${PERL5LIB:+:$PERL5LIB}
 ENV LD_LIBRARY_PATH=${MINC_TOOLKIT}/lib:${MINC_TOOLKIT}/lib/InsightToolkit${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
 ENV MNI_DATAPATH=${MINC_TOOLKIT}/../share:${MINC_TOOLKIT}/share
 ENV MINC_FORCE_V2=1
 ENV MINC_COMPRESS=4
 ENV VOLUME_CACHE_THRESHOLD=-1
 ENV MANPATH=${MINC_TOOLKIT}/man${MANPATH:+:$MANPATH}
 ENV ANTSPATH=${MINC_TOOLKIT}/bin
 
 # Download MINC Toolkit auxiliary packages
@@ -119,6 +111,8 @@
 ENV TMPDIR=/tmp
 ENV LORIS_CONFIG=/opt/${PROJECT}/bin/mri/dicom-archive
 ENV LORIS_MRI=/opt/${PROJECT}/bin/mri
 ENV PYTHONPATH=$PYTHONPATH:/opt/${PROJECT}/bin/mri/python:/opt/${PROJECT}/bin/mri/python/react-series-data-viewer
 ENV BEASTLIB=${MINC_TOOLKIT_DIR}/../share/beast-library-1.1
 ENV MNI_MODELS=${MINC_TOOLKIT_DIR}/../share/icbm152_model_09c
+
+ENTRYPOINT ["./test/entrypoint.sh"]