Merge branch 'enh/update-zenodo-sorter' into rel/1.2.3

Author: oesteban

Diff for: docker/generate_dockerfiles.sh

@@ -68,10 +68,11 @@ function generate_base_dockerfile() {
     --spm12 version=r7219 \
     --env 'LD_LIBRARY_PATH=/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH' \
     --freesurfer version=6.0.0-min \
+    --dcm2niix version=v1.0.20190902 method=source \
     --run 'echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tCjUxNzIKICpDdnVtdkVWM3pUZmcKRlM1Si8yYzFhZ2c0RQoiID4gL29wdC9mcmVlc3VyZmVyLTYuMC4wLW1pbi9saWNlbnNlLnR4dA==" | base64 -d | sh' \
     --install afni ants apt-utils bzip2 convert3d file fsl-core \
               fsl-mni152-templates fusefat g++ git graphviz make python ruby \
-              unzip xvfb \
+              unzip xvfb git-annex-standalone liblzma-dev \
     --add-to-entrypoint "source /etc/fsl/fsl.sh && source /etc/afni/afni.sh" \
     --env ANTSPATH='/usr/lib/ants' \
           PATH='/usr/lib/ants:$PATH' \

Diff for: nipype/info.py

@@ -182,6 +182,7 @@ def get_nipype_gitversion():
 ]
 
 EXTRA_REQUIRES = {
+    'data': ['datalad'],
     'doc': ['Sphinx>=1.4', 'numpydoc', 'matplotlib', 'pydotplus', 'pydot>=1.2.3'],
     'duecredit': ['duecredit'],
     'nipy': ['nitime', 'nilearn<0.5.0', 'dipy', 'nipy', 'matplotlib'],

Diff for: nipype/interfaces/afni/model.py

@@ -541,6 +541,10 @@ class RemlfitInputSpec(AFNICommandInputSpec):
         desc='turns on more progress messages, including memory usage '
         'progress reports at various stages',
         argstr='-verb')
+    goforit = traits.Bool(
+        desc='With potential issues flagged in the design matrix, an attempt '
+        'will nevertheless be made to fit the model',
+        argstr='-GOFORIT')
     ovar = File(
         desc='dataset for OLSQ st.dev. parameter (kind of boring)',
         argstr='-Ovar %s')

Diff for: nipype/interfaces/dcm2nii.py

@@ -7,6 +7,8 @@
 import os
 import re
 from copy import deepcopy
+import itertools as it
+from glob import iglob
 
 from ..utils.filemanip import split_filename
 from .base import (CommandLine, CommandLineInputSpec, InputMultiPath, traits,
@@ -328,7 +330,7 @@ class Dcm2niixInputSpec(CommandLineInputSpec):
         False,
         argstr='-t',
         usedefault=True,
-        desc="Flag if text notes include private patient details")
+        desc="Text notes including private patient details")
     compression = traits.Enum(
         1, 2, 3, 4, 5, 6, 7, 8, 9,
         argstr='-%d',
@@ -346,6 +348,9 @@ class Dcm2niixInputSpec(CommandLineInputSpec):
     philips_float = traits.Bool(
         argstr='-p',
         desc="Philips precise float (not display) scaling")
+    to_nrrd = traits.Bool(
+        argstr="-e",
+        desc="Export as NRRD instead of NIfTI")
 
 
 class Dcm2niixOutputSpec(TraitedSpec):
@@ -393,8 +398,11 @@ def version(self):
         return Info.version()
 
     def _format_arg(self, opt, spec, val):
-        bools = ['bids_format', 'merge_imgs', 'single_file', 'verbose', 'crop',
-                 'has_private', 'anon_bids', 'ignore_deriv', 'philips_float']
+        bools = [
+            'bids_format', 'merge_imgs', 'single_file', 'verbose', 'crop',
+            'has_private', 'anon_bids', 'ignore_deriv', 'philips_float',
+            'to_nrrd',
+        ]
         if opt in bools:
             spec = deepcopy(spec)
             if val:
@@ -410,52 +418,54 @@ def _run_interface(self, runtime):
         # may use return code 1 despite conversion
         runtime = super(Dcm2niix, self)._run_interface(
             runtime, correct_return_codes=(0, 1, ))
-        if self.inputs.bids_format:
-            (self.output_files, self.bvecs, self.bvals,
-             self.bids) = self._parse_stdout(runtime.stdout)
-        else:
-            (self.output_files, self.bvecs, self.bvals) = self._parse_stdout(
-                runtime.stdout)
+        self._parse_files(self._parse_stdout(runtime.stdout))
         return runtime
 
     def _parse_stdout(self, stdout):
-        files = []
-        bvecs = []
-        bvals = []
-        bids = []
-        skip = False
-        find_b = False
+        filenames = []
         for line in stdout.split("\n"):
-            if not skip:
-                out_file = None
-                if line.startswith("Convert "):  # output
-                    fname = str(re.search('\S+/\S+', line).group(0))
-                    out_file = os.path.abspath(fname)
-                    # extract bvals
-                    if find_b:
-                        bvecs.append(out_file + ".bvec")
-                        bvals.append(out_file + ".bval")
-                        find_b = False
-                # next scan will have bvals/bvecs
-                elif 'DTI gradients' in line or 'DTI gradient directions' in line or 'DTI vectors' in line:
-                    find_b = True
-                if out_file:
-                    ext = '.nii' if self.inputs.compress == 'n' else '.nii.gz'
-                    files.append(out_file + ext)
-                    if self.inputs.bids_format:
-                        bids.append(out_file + ".json")
-            skip = False
-        # just return what was done
-        if not bids:
-            return files, bvecs, bvals
+            if line.startswith("Convert "):  # output
+                fname = str(re.search(r'\S+/\S+', line).group(0))
+                filenames.append(os.path.abspath(fname))
+        return filenames
+
+    def _parse_files(self, filenames):
+        outfiles, bvals, bvecs, bids = [], [], [], []
+        outtypes = [".bval", ".bvec", ".json", ".txt"]
+        if self.inputs.to_nrrd:
+            outtypes += [".nrrd", ".nhdr", ".raw.gz"]
         else:
-            return files, bvecs, bvals, bids
+            outtypes += [".nii", ".nii.gz"]
+
+        for filename in filenames:
+            # search for relevant files, and sort accordingly
+            for fl in search_files(filename, outtypes):
+                if (
+                    fl.endswith(".nii") or
+                    fl.endswith(".gz") or
+                    fl.endswith(".nrrd") or
+                    fl.endswith(".nhdr")
+                ):
+                    outfiles.append(fl)
+                elif fl.endswith(".bval"):
+                    bvals.append(fl)
+                elif fl.endswith(".bvec"):
+                    bvecs.append(fl)
+                elif fl.endswith(".json") or fl.endswith(".txt"):
+                    bids.append(fl)
+        self.output_files = outfiles
+        self.bvecs = bvecs
+        self.bvals = bvals
+        self.bids = bids
 
     def _list_outputs(self):
         outputs = self.output_spec().get()
         outputs['converted_files'] = self.output_files
         outputs['bvecs'] = self.bvecs
         outputs['bvals'] = self.bvals
-        if self.inputs.bids_format:
-            outputs['bids'] = self.bids
+        outputs['bids'] = self.bids
         return outputs
+
+# https://stackoverflow.com/a/4829130
+def search_files(prefix, outtypes):
+    return it.chain.from_iterable(iglob(prefix + outtype) for outtype in outtypes)

Diff for: nipype/interfaces/tests/test_auto_Dcm2niix.py

@@ -61,6 +61,7 @@ def test_Dcm2niix_inputs():
             position=-1,
             xor=['source_dir'],
         ),
+        to_nrrd=dict(argstr='-e', ),
         verbose=dict(
             argstr='-v',
             usedefault=True,

Diff for: nipype/interfaces/tests/test_extra_dcm2nii.py

@@ -41,17 +41,15 @@ def assert_dwi(eg, bids):
             # ensure all outputs are of equal lengths
             assert len(set(map(len, outputs))) == 1
         else:
-            assert not eg2.outputs.bids
+            assert not eg.outputs.bids
 
     dcm = Dcm2niix()
     dcm.inputs.source_dir = datadir
     dcm.inputs.out_filename = '%u%z'
-    eg1 = dcm.run()
-    assert_dwi(eg1, True)
+    assert_dwi(dcm.run(), True)
 
     # now run specifying output directory and removing BIDS option
     outdir = tmpdir.mkdir('conversion').strpath
     dcm.inputs.output_dir = outdir
     dcm.inputs.bids_format = False
-    eg2 = dcm.run()
-    assert_dwi(eg2, False)
+    assert_dwi(dcm.run(), False)

Diff for: tools/update_zenodo.py

@@ -1,46 +1,16 @@
 #!/usr/bin/env python3
+"""Update and sort the creators list of the zenodo record."""
+import sys
+import shutil
+from pathlib import Path
 import json
 from fuzzywuzzy import fuzz, process
-import shutil
-import os
 import subprocess as sp
 
-if os.path.exists('line-contributions.txt'):
-    with open('line-contributions.txt', 'rt') as fp:
-        lines = fp.readlines()
-else:
-    if shutil.which('git-line-summary'):
-        print("Running git-line-summary on nipype repo")
-        lines = sp.check_output(['git-line-summary']).decode().split('\n')
-    else:
-        raise RuntimeError("Install Git Extras to view git contributors")
-
-data = [' '.join(line.strip().split()[1:-1]) for line in lines if '%' in line]
-
-# load zenodo from master
-with open('.zenodo.json', 'rt') as fp:
-    zenodo = json.load(fp)
-zen_names = [' '.join(val['name'].split(',')[::-1]).strip()
-             for val in zenodo['creators']]
-
-name_matches = []
-
-for ele in data:
-    matches = process.extract(ele, zen_names, scorer=fuzz.token_sort_ratio,
-                              limit=2)
-    # matches is a list [('First match', % Match), ('Second match', % Match)]
-    if matches[0][1] > 80:
-        val = zenodo['creators'][zen_names.index(matches[0][0])]
-    else:
-        # skip unmatched names
-        print("No entry to sort:", ele)
-        continue
-
-    if val not in name_matches:
-        name_matches.append(val)
-
+# These ORCIDs should go last
+CREATORS_LAST = ['Gorgolewski, Krzysztof J.', 'Ghosh, Satrajit']
 # for entries not found in line-contributions
-missing_entries = [
+MISSING_ENTRIES = [
     {"name": "Varada, Jan"},
     {"name": "Schwabacher, Isaac"},
     {"affiliation": "Child Mind Institute / Nathan Kline Institute",
@@ -60,31 +30,62 @@
     {"name": "Lai, Jeff"}
 ]
 
-for entry in missing_entries:
-    name_matches.append(entry)
+if __name__ == '__main__':
+    contrib_file = Path('line-contributors.txt')
+    lines = []
+    if contrib_file.exists():
+        print('WARNING: Reusing existing line-contributors.txt file.', file=sys.stderr)
+        lines = contrib_file.read_text().splitlines()
 
-
-def fix_position(creators):
-    # position first / last authors
-    f_authr = None
-    l_authr = None
-
-    for i, info in enumerate(creators):
-        if info['name'] == 'Gorgolewski, Krzysztof J.':
-            f_authr = i
-        if info['name'] == 'Ghosh, Satrajit':
-            l_authr = i
-
-    if f_authr is None or l_authr is None:
-        raise AttributeError('Missing important people')
-
-    creators.insert(0, creators.pop(f_authr))
-    creators.insert(len(creators), creators.pop(l_authr + 1))
-    return creators
-
-
-zenodo['creators'] = fix_position(name_matches)
-
-with open('.zenodo.json', 'wt') as fp:
-    json.dump(zenodo, fp, indent=2, sort_keys=True)
-    fp.write('\n')
+    git_line_summary_path = shutil.which('git-line-summary')
+    if not lines and git_line_summary_path:
+        print("Running git-line-summary on nipype repo")
+        lines = sp.check_output([git_line_summary_path]).decode().splitlines()
+        contrib_file.write_text('\n'.join(lines))
+
+    if not lines:
+        raise RuntimeError("""\
+Could not find line-contributors from git repository.%s""" % """ \
+git-line-summary not found, please install git-extras. """ * (git_line_summary_path is None))
+
+    data = [' '.join(line.strip().split()[1:-1]) for line in lines if '%' in line]
+
+    # load zenodo from master
+    zenodo_file = Path('.zenodo.json')
+    zenodo = json.loads(zenodo_file.read_text())
+    zen_names = [' '.join(val['name'].split(',')[::-1]).strip()
+                 for val in zenodo['creators']]
+    total_names = len(zen_names) + len(MISSING_ENTRIES)
+
+    name_matches = []
+    position = 1
+    for ele in data:
+        matches = process.extract(ele, zen_names, scorer=fuzz.token_sort_ratio,
+                                  limit=2)
+        # matches is a list [('First match', % Match), ('Second match', % Match)]
+        if matches[0][1] > 80:
+            val = zenodo['creators'][zen_names.index(matches[0][0])]
+        else:
+            # skip unmatched names
+            print("No entry to sort:", ele)
+            continue
+
+        if val not in name_matches:
+            if val['name'] not in CREATORS_LAST:
+                val['position'] = position
+                position += 1
+            else:
+                val['position'] = total_names + CREATORS_LAST.index(val['name'])
+            name_matches.append(val)
+
+    for missing in MISSING_ENTRIES:
+        missing['position'] = position
+        position += 1
+        name_matches.append(missing)
+
+    zenodo['creators'] = sorted(name_matches, key=lambda k: k['position'])
+    # Remove position
+    for creator in zenodo['creators']:
+        del creator['position']
+
+    zenodo_file.write_text('%s\n' % json.dumps(zenodo, indent=2, sort_keys=True))