merging in alleninstitute

Author: fcollman

CONTRIBUTING.md

@@ -0,0 +1,26 @@
+# Allen Institute Contribution Agreement
+
+This document describes the terms under which you may make “Contributions” — 
+which may include without limitation, software additions, revisions, bug fixes, configuration changes,
+documentation, or any other materials — to any of the projects owned or managed by the Allen Institute.
+If you have questions about these terms, please contact us at [email protected].  
+
+You certify that:
+
+•	Your Contributions are either:
+
+1.	Created in whole or in part by you and you have the right to submit them under the designated license 
+(described below); or
+2.	Based upon previous work that, to the best of your knowledge, is covered under an appropriate 
+open source license and you have the right under that license to submit that work with modifications,
+whether created in whole or in part by you, under the designated license; or
+
+3.	Provided directly to you by some other person who certified (1) or (2) and you have not modified them.
+
+•	You are granting your Contributions to the Allen Institute under the terms of the [2-Clause BSD license](https://opensource.org/licenses/BSD-2-Clause)
+(the “designated license”).
+
+•	You understand and agree that the Allen Institute projects and your Contributions are public and that 
+a record of the Contributions (including all metadata and personal information you submit with them) is 
+maintained indefinitely and may be redistributed consistent with the Allen Institute’s mission and the 
+2-Clause BSD license.  

Dockerfile

@@ -8,6 +8,7 @@ RUN pip install setuptools --upgrade --disable-pip-version-check
 RUN pip install argschema --upgrade --disable-pip-version-check
 RUN pip install jupyter
 RUN apt-get install libspatialindex-dev -y
+RUN conda install nomkl
 COPY . /usr/local/render-python-apps
 
 #RUN git clone https://github.com/fcollman/render-python-apps

allen_utils/docker_setup.sh

@@ -1,5 +1,5 @@
-docker pull atbigdawg:5000/fcollman/render-python:latest
-docker tag atbigdawg:5000/fcollman/render-python:latest fcollman/render-python:latest
+#docker pull atbigdawg:5000/fcollman/render-python:latest
+#docker tag atbigdawg:5000/fcollman/render-python:latest fcollman/render-python:latest
 docker build -t fcollman/render-python-apps .
 docker tag fcollman/render-python-apps atbigdawg:5000/fcollman/render-python-apps
 docker push atbigdawg:5000/fcollman/render-python-apps
@@ -10,14 +10,11 @@ docker run -t --name renderapps \
 -v /nas2:/nas2 \
 -v /nas3:/nas3 \
 -v /nas4:/nas4 \
+-v /nas5:/nas5 \
 -v /data:/data \
 -v /pipeline:/pipeline \
 -v /pipeline/render-python-apps:/usr/local/render-python-apps \
 -v /etc/hosts:/etc/hosts \
---dns 10.128.104.10 \
--p 8888:8888 \
+-p 7777:7777 \
 -e "PASSWORD=$JUPYTERPASSWORD" \
--i -t fcollman/render-python-apps \
-/bin/bash -c "/opt/conda/bin/jupyter notebook --config=/root/.jupyter/jupyter_notebook_config.py --notebook-dir=/pipeline/render-python-apps --no-browser --allow-root" 
- 
-
+-i -t fcollman/render-python-apps 

allen_utils/reorg_setup.sh

@@ -1,21 +1,20 @@
-docker pull atbigdawg:5000/fcollman/render-python:master
-docker tag atbigdawg:5000/fcollman/render-python:master fcollman/render-python
-docker build -t fcollman/render-python-apps:reorg .
-docker tag fcollman/render-python-apps:reorg fcollman/render-python-apps:reorg_testsharmi
-docker push atbigdawg:5000/fcollman/render-python-apps
-docker kill renderapps_testsharmi
-docker rm renderapps_testsharmi
-docker run -d --name renderapps_testsharmi \
+docker pull localhost:5000/fcollman/render-python:master
+docker tag localhost:5000/fcollman/render-python:master fcollman/render-python
+docker build -t sharmi/render-python-apps:reorg .
+#docker tag fcollman/render-python-apps:reorg fcollman/render-python-apps:reorg_testsharmi
+#docker push atbigdawg:5000/fcollman/render-python-apps
+docker kill renderapps_develop
+docker rm renderapps_develop
+docker run -d --name renderapps_develop \
 -v /nas2:/nas2 \
 -v /nas:/nas \
 -v /nas3:/nas3 \
 -v /nas4:/nas4 \
 -v /data:/data \
 -v /pipeline:/pipeline \
--v /pipeline/sharmi/Sharmi_tools/render-python-apps:/usr/local/render-python-apps \
+-v /pipeline/sharmi/Sharmi_tools/render-python-apps-branches/DEVELOP/render-python-apps:/usr/local/share/render-python-apps \
 -v /etc/hosts:/etc/hosts \
--p 9999:9999 \
+-p 7777:7777 \
 -e "PASSWORD=$JUPYTERPASSWORD" \
--i -t fcollman/render-python-apps:reorg_testsharmi
- 
-
+-i -t sharmi/render-python-apps:reorg \
+/bin/bash -c "jupyter notebook --config=/root/.jupyter/jupyter_notebook_config.py  --no-browser --allow-root --notebook-dir=/pipeline/sharmi/Sharmi_tools/render-python-apps-branches/DEVELOP/render-python-apps"

jupyter_notebook_config.py

@@ -16,7 +16,7 @@
 from IPython.lib import passwd
 
 c.NotebookApp.ip = '*'
-c.NotebookApp.port = 8888
+c.NotebookApp.port = 7777
 c.NotebookApp.open_browser = False
 c.MultiKernelManager.default_kernel_name = 'python2'
 

renderapps/TrakEM2/ImportTrakEM2Annotations.py

@@ -174,13 +174,14 @@ def run(self):
             pot_render_tilespecs = self.render.run(renderapi.tilespec.get_tile_specs_from_z,
                                                    self.args['EMstack'],
                                                    ts.z)
-            mml=ts.ip.get(0,None)
-            if mml is None:
-                mml = ts.channels[0].ip[0]
+            try:
+                mml=ts.ip.get(0)
+            except KeyError:
+                mml = ts.channels[0].ip.get(0)
 
 
-            filepath = (os.path.split(mml.imageUrl)[1]).split('_flip')[0]
-            pot_filepaths = [(os.path.split(t.ip[0].imageUrl)[1]).split(
+            filepath = (os.path.split(mml['imageUrl'])[1]).split('_flip')[0]
+            pot_filepaths = [(os.path.split(t.ip.get(0)['imageUrl'])[1]).split(
                 '_flip')[0] for t in pot_render_tilespecs]
             render_tilespecs.append(next(t for t, fp in zip(
                 pot_render_tilespecs, pot_filepaths) if fp == filepath))

renderapps/TrakEM2/MakeSynaptogramsFromAnnotation_OidList.py

@@ -17,8 +17,7 @@
 ############## this is modified from notebook: http://ibs-forrestc-ux1:8888/notebooks/MakeSynaptogramsFromAnnotations.ipynb #####
 
 
-example_parameters = 
-{
+example_parameters = {
     "render":{
         "host": "ibs-forrestc-ux1",
         "port": 80,
@@ -27,51 +26,27 @@
         "client_scripts": "/pipeline/render/render-ws-java-client/src/main/scripts"
     },
 
-    "global_annotation_file":"/nas3/data/M247514_Rorb_1/annotation/m247514_Take2Site4Annotation_MN_Take2Site4global.json",
-    "annotation_metadata_json":"/nas3/data/M247514_Rorb_1/annotation/junk3.json",
-    "fig_directory":"/nas3/data/M247514_Rorb_1/processed/Take2Site4Missing",
-    "synapses_to_make":['1126',
-                        '1276',
-                        '1302',
-                        '1352',
-                        '1380',
-                        '1386',
-                        '1490',
-                        '1510',
-                        '1554',
-                        '2693',
-                        '2695',
-                        '2697',
-                        '2699',
-                        '2701',
-                        '2703',
-                        '2705',
-                        '2707'],   
+    "global_annotation_file":"/nas/data/M246930_Scnn1a_4_f1/annotation/m246930_site5_annotation_MN_adjustZ_bb_Take2Site5_EMA_global.json",
+    "annotation_metadata_json":"/nas/data/M246930_Scnn1a_4_f1/SEMdata/processed/Synaptograms/annotationMetadata/m246930_Take2Site5_synaptogram_metadata.json",
+    "fig_directory":"/nas/data/M246930_Scnn1a_4_f1/SEMdata/processed/Synaptograms/Take2Site5_TdTeSyn",
+    "synapses_to_make":['1226','1326','1416','2171','2299','668','2707','2709','2711'],   
     "channel_stacks":[
                      {
-                            "stack":"Take2Site4Align_EMclahe"
+ 
+                            "stack":"EMSite5_take2_EMA"
                         },
                         {
-                            "stack":"Take2Site4Align_EMclahe"
+
+                            "stack":"EMSite5_take2_EMA"
                         },
                         {
                             "channel":"TdTomato",
-                            "stack":"Take2Site4Align_Session2",
-                            "maxIntensity":25000
-                        },
-                        {
-                            "channel":"PSD95",
-                            "stack":"Take2Site4Align_Session1",
+                            "stack":"Take2Site5_EMA_STI_DCV_FF_allSession_2",
                             "maxIntensity":10000
                         },
-                        {
-                            "channel":"GABA",
-                            "stack":"Take2Site4Align_Session3",
-                            "maxIntensity":5000
-                        },
                         {
                             "channel":"synapsin",
-                            "stack":"Take2Site4Align_Session3",
+                            "stack":"Take2Site5_manReg_EMA_STI_DCV_FF_allSession_3",
                             "maxIntensity":16000
                         }
                     ]
@@ -104,15 +79,14 @@ class MakeAnnotationSynaptogramParameters(RenderParameters):
 
 
 
-    def load_annotation_file(annotation_path):
+def load_annotation_file(annotation_path):
     with open(annotation_path,'r') as fp:
             annotation_d = json.load(fp)
     schema = AnnotationFile()
     annotations,errors = schema.load(annotation_d)        
     assert(len(errors)==0)
     return annotations
 
-
 def make_synaptogram(render,channel_stacks,savedir,al,border=400/3,borderz=2):
     oid = al['oid']
     zvalues = []
@@ -162,7 +136,8 @@ def make_synaptogram(render,channel_stacks,savedir,al,border=400/3,borderz=2):
     plt.rcParams['axes.facecolor'] = 'black'
     f,ax=plt.subplots(Nc,Nz,
                       figsize=(fig_width,int(fig_width*ratio)),
-                      gridspec_kw = {'wspace':0, 'hspace':0})
+                      gridspec_kw = {'wspace':0, 'hspace':0},
+                     facecolor='black')
     plt.subplots_adjust(left=0.0, right=1.0, top=1.0, bottom=0.0)
     for c,chstack in enumerate(channel_stacks):
         for zi,z in enumerate(zvals):
@@ -188,11 +163,11 @@ def make_synaptogram(render,channel_stacks,savedir,al,border=400/3,borderz=2):
         zi = int(area['z']-minZ)
         for c in range(1,len(channel_stacks)):
             a = ax[c,zi]
-            a.plot(area['global_path'][:,0],area['global_path'][:,1],c='g',linewidth=3)
+            a.plot(area['global_path'][:,0],area['global_path'][:,1],c='Teal',linewidth=2)
     for c,chstack in enumerate(channel_stacks):
         chname = chstack.get('channel',chstack['stack'])
 
-        ax[c,0].text(minX,minY,chname,fontdict={'color':'w','weight':'bold'},fontsize=16)
+        ax[c,0].text(minX,minY,chname[0:4],fontdict={'color':'SLATEGRAY','weight':'bold'},fontsize=14)
     #f.tight_layout(True)
     fname = os.path.join(savedir,'{}.png'.format(oid))
     f.savefig(fname)
@@ -202,6 +177,7 @@ def make_synaptogram(render,channel_stacks,savedir,al,border=400/3,borderz=2):
     del(f)
     return d
 
+
 class MakeAnnotationSynaptograms(RenderModule):
     default_schema = MakeAnnotationSynaptogramParameters
 

renderapps/TrakEM2/transform_local_annotation_json_to_global.py

@@ -12,14 +12,14 @@
 example_input={
     "render":{
         "host":"ibs-forrestc-ux1",
-        "port":8080,
+        "port":80,
         "owner":"Forrest",
-        "project":"M247514_Rorb_1",
+        "project":"M246930_Scnn1a_4_f1",
         "client_scripts":"/pipeline/render/render-ws-java-client/src/main/scripts"
     },
-    "stack":"Take2Site4Align_EMclahe",
-    "input_annotation_file":"/nas3/data/M247514_Rorb_1/annotation/m247514_Take2Site4Annotation_MN_bb_local.json",
-   "output_annotation_file":"/nas3/data/M247514_Rorb_1/annotation/m247514_Take2Site4Annotation_MN_bb_Take2Site4Align_EMclahe_global.json"
+    "stack":"EMSite5_take2_EMA",
+    "input_annotation_file":"/nas3/data/M246930_Scnn1a_4_f1/annotation/m246930_site5_annotation_MN_adjustZ_bb_local.json",
+    "output_annotation_file":"/nas3/data/M246930_Scnn1a_4_f1/annotation/m246930_site5_annotation_MN_adjustZ_bb_EMSite5_take2_EMA_global.json"
 }
 
 

renderapps/__init__.py

@@ -15,8 +15,9 @@
 from . import wrinkle_detection
 from . import rough_align
 from . import synapse_detection
+from . import intensity_correction
 
 __all__ = ['cross_modal_registration', 'dataimport', 'materialize', 'pointmatch',
            'module','shapely',
            'registration', 'section_polygons', 'stack',
-           'stitching', 'tile', 'TrakEM2','transfer','wrinkle_detection','datamanagement','rough_align','synapse_detection']
+           'stitching', 'tile', 'TrakEM2','transfer','wrinkle_detection','datamanagement','rough_align','synapse_detection','intensity_correction']

renderapps/cross_modal_registration/import_EM_registration_projects_multi.py

@@ -18,16 +18,16 @@
 example_json = {
     "render":{
         "host":"ibs-forrestc-ux1",
-        "port":8080,
+        "port":8988,
         "owner":"Forrest",
-        "project":"M247514_Rorb_1",
+        "project":"M246930_Scnn1a_4_f1",
         "client_scripts":"/pipeline/render/render-ws-java-client/src/main/scripts"
     },
-    "inputStack":"EM_Site4_stitched_SHIFT",
-    "LMstacks":["BIGREG_MARCH_21_MBP_deconvnew"],
-    "outputStack":"BIGREG_EM_Site4_stitched",
+    "inputStack":"REG_STI_FF_S03_DAPI_3",
+    "LMstacks":["Stitched_1_DAPI_1"],
+    "outputStack":"REG01_STI_FF_S03_DAPI_3",
     "renderHome":"/var/www/render",
-    "outputXMLdir":"/nas3/data/M247514_Rorb_1/processed/EMLMRegProjects_Site4/"
+    "outputXMLdir":"/nas/data/M246930_Scnn1a_4_f1/processed/tilepairfiles/TrackEM_projects/"
 }
 class ImportEMRegistrationMultiProjects(TrakEM2RenderModule):
     def __init__(self,schema_type=None,*args,**kwargs):
@@ -47,14 +47,27 @@ def run(self):
         EMz = renderapi.stack.get_z_values_for_stack(self.args['inputStack'],render=self.render)
 
         tilespecsfiles = []
-        shiftTransform = AffineModel(B0=self.args['minX'],B1=self.args['minY'])
+
+        buffersize = self.args['buffersize']
+        self.args['minX'] = self.args['minX'] - buffersize
+        self.args['minY'] = self.args['minY'] - buffersize
+        self.args['maxX'] = self.args['maxX'] + buffersize
+        self.args['maxY'] = self.args['maxY'] + buffersize
+        #width = self.args['maxX']-self.args['minX']
+        #height = self.args['maxY']-self.args['minY']
+
+        print("This is buffersize: %d "%buffersize)
+
+        shiftTransform = AffineModel(B0=self.args['minX'] ,B1=self.args['minY'] )
+
+
 
         for z in EMz:
             infile = os.path.join(xmlDir,'%05d.xml'%z)
             outfile = os.path.join(xmlDir,'%05d.json'%z)
             newoutfile = os.path.join(xmlDir,'%05d-new.json'%z)
             self.convert_trakem2_project(infile,xmlDir,outfile)
-         
+
             newtilejson = json.load(open(outfile,'r'))
             newEMtilespecs = [TileSpec(json=tsj) for tsj in newtilejson]
             EMtilespecs = renderapi.tilespec.get_tile_specs_from_minmax_box(
@@ -81,4 +94,4 @@ def run(self):
 
 if __name__ == "__main__":
     mod = ImportEMRegistrationMultiProjects(input_data= example_json)
-    mod.run()
+    mod.run()

renderapps/cross_modal_registration/import_LM_subset_from_EM_registration_multi.py

@@ -1,3 +1,6 @@
+if __name__ == "__main__" and __package__ is None:
+    __package__ = "renderapps.cross_modal_registration.import_LM_subset_from_EM_registration_multi"
+
 import renderapi
 from ..TrakEM2.trakem2utils import \
     createchunks, createheader, createproject, \

renderapps/cross_modal_registration/make_EM_LM_registration_projects.py

@@ -86,7 +86,7 @@ def run(self):
                             mod.args['minY'],
                             mod.args['maxY'],
                             render=mod.render)
-            createlayer_fromtilespecs(LMtilespecs, outfile,1,shiftx=-mod.args['minX'],shifty=-mod.args['minY'],affineOnly=False)
+            createlayer_fromtilespecs(LMtilespecs, outfile,1,shiftx=-mod.args['minX'],shifty=-mod.args['minY'],affineOnly=True)
             createfooters(outfile)
             print(self.args)