Merge pull request #30 from AllenInstitute/forrest_leila_changes

forrest leila changes for cross modal registration

Author: fcollman

allen_utils/docker_setup.sh

@@ -1,5 +1,5 @@
-docker pull atbigdawg:5000/fcollman/render-python:latest
-docker tag atbigdawg:5000/fcollman/render-python:latest fcollman/render-python:latest
+#docker pull atbigdawg:5000/fcollman/render-python:latest
+#docker tag atbigdawg:5000/fcollman/render-python:latest fcollman/render-python:latest
 docker build -t fcollman/render-python-apps .
 docker tag fcollman/render-python-apps atbigdawg:5000/fcollman/render-python-apps
 docker push atbigdawg:5000/fcollman/render-python-apps
@@ -10,14 +10,11 @@ docker run -t --name renderapps \
 -v /nas2:/nas2 \
 -v /nas3:/nas3 \
 -v /nas4:/nas4 \
+-v /nas5:/nas5 \
 -v /data:/data \
 -v /pipeline:/pipeline \
 -v /pipeline/render-python-apps:/usr/local/render-python-apps \
 -v /etc/hosts:/etc/hosts \
---dns 10.128.104.10 \
--p 8888:8888 \
+-p 7777:7777 \
 -e "PASSWORD=$JUPYTERPASSWORD" \
--i -t fcollman/render-python-apps \
-/bin/bash -c "/opt/conda/bin/jupyter notebook --config=/root/.jupyter/jupyter_notebook_config.py --notebook-dir=/pipeline/render-python-apps --no-browser --allow-root" 
- 
-
+-i -t fcollman/render-python-apps 

allen_utils/reorg_setup.sh

@@ -1,21 +1,20 @@
-docker pull atbigdawg:5000/fcollman/render-python:master
-docker tag atbigdawg:5000/fcollman/render-python:master fcollman/render-python
-docker build -t fcollman/render-python-apps:reorg .
-docker tag fcollman/render-python-apps:reorg fcollman/render-python-apps:reorg_testsharmi
-docker push atbigdawg:5000/fcollman/render-python-apps
-docker kill renderapps_testsharmi
-docker rm renderapps_testsharmi
-docker run -d --name renderapps_testsharmi \
+docker pull localhost:5000/fcollman/render-python:master
+docker tag localhost:5000/fcollman/render-python:master fcollman/render-python
+docker build -t sharmi/render-python-apps:reorg .
+#docker tag fcollman/render-python-apps:reorg fcollman/render-python-apps:reorg_testsharmi
+#docker push atbigdawg:5000/fcollman/render-python-apps
+docker kill renderapps_develop
+docker rm renderapps_develop
+docker run -d --name renderapps_develop \
 -v /nas2:/nas2 \
 -v /nas:/nas \
 -v /nas3:/nas3 \
 -v /nas4:/nas4 \
 -v /data:/data \
 -v /pipeline:/pipeline \
--v /pipeline/sharmi/Sharmi_tools/render-python-apps:/usr/local/render-python-apps \
+-v /pipeline/sharmi/Sharmi_tools/render-python-apps-branches/DEVELOP/render-python-apps:/usr/local/share/render-python-apps \
 -v /etc/hosts:/etc/hosts \
--p 9999:9999 \
+-p 7777:7777 \
 -e "PASSWORD=$JUPYTERPASSWORD" \
--i -t fcollman/render-python-apps:reorg_testsharmi
- 
-
+-i -t sharmi/render-python-apps:reorg \
+/bin/bash -c "jupyter notebook --config=/root/.jupyter/jupyter_notebook_config.py  --no-browser --allow-root --notebook-dir=/pipeline/sharmi/Sharmi_tools/render-python-apps-branches/DEVELOP/render-python-apps"

jupyter_notebook_config.py

@@ -16,7 +16,7 @@
 from IPython.lib import passwd
 
 c.NotebookApp.ip = '*'
-c.NotebookApp.port = 8888
+c.NotebookApp.port = 7777
 c.NotebookApp.open_browser = False
 c.MultiKernelManager.default_kernel_name = 'python2'
 

renderapps/TrakEM2/ImportTrakEM2Annotations.py

@@ -174,13 +174,14 @@ def run(self):
             pot_render_tilespecs = self.render.run(renderapi.tilespec.get_tile_specs_from_z,
                                                    self.args['EMstack'],
                                                    ts.z)
-            mml=ts.ip.get(0,None)
-            if mml is None:
-                mml = ts.channels[0].ip[0]
+            try:
+                mml=ts.ip.get(0)
+            except KeyError:
+                mml = ts.channels[0].ip.get(0)
 
 
-            filepath = (os.path.split(mml.imageUrl)[1]).split('_flip')[0]
-            pot_filepaths = [(os.path.split(t.ip[0].imageUrl)[1]).split(
+            filepath = (os.path.split(mml['imageUrl'])[1]).split('_flip')[0]
+            pot_filepaths = [(os.path.split(t.ip.get(0)['imageUrl'])[1]).split(
                 '_flip')[0] for t in pot_render_tilespecs]
             render_tilespecs.append(next(t for t, fp in zip(
                 pot_render_tilespecs, pot_filepaths) if fp == filepath))

renderapps/__init__.py

@@ -15,8 +15,9 @@
 from . import wrinkle_detection
 from . import rough_align
 from . import synapse_detection
+from . import intensity_correction
 
 __all__ = ['cross_modal_registration', 'dataimport', 'materialize', 'pointmatch',
            'module','shapely',
            'registration', 'section_polygons', 'stack',
-           'stitching', 'tile', 'TrakEM2','transfer','wrinkle_detection','datamanagement','rough_align','synapse_detection']
+           'stitching', 'tile', 'TrakEM2','transfer','wrinkle_detection','datamanagement','rough_align','synapse_detection','intensity_correction']

renderapps/cross_modal_registration/import_EM_registration_projects_multi.py

@@ -50,9 +50,9 @@ def run(self):
 
         buffersize = self.args['buffersize']
         self.args['minX'] = self.args['minX'] - buffersize
-    	self.args['minY'] = self.args['minY'] - buffersize
-    	self.args['maxX'] = self.args['maxX'] + buffersize
-    	self.args['maxY'] = self.args['maxY'] + buffersize
+        self.args['minY'] = self.args['minY'] - buffersize
+        self.args['maxX'] = self.args['maxX'] + buffersize
+        self.args['maxY'] = self.args['maxY'] + buffersize
         #width = self.args['maxX']-self.args['minX']
         #height = self.args['maxY']-self.args['minY']
 

renderapps/cross_modal_registration/make_EM_LM_registration_projects_multi.py

@@ -50,7 +50,7 @@ def run(self):
 
 
 	#buffersize = 3000
-    buffersize = self.args['buffersize']
+        buffersize = self.args['buffersize']
 	self.args['minX'] = self.args['minX'] - buffersize
 	self.args['minY'] = self.args['minY'] - buffersize
 	self.args['maxX'] = self.args['maxX'] + buffersize

renderapps/module/render_module.py

@@ -83,6 +83,8 @@ class EMLMRegistrationMultiParameters(TEM2ProjectTransfer):
         required=False, description='maximum z (default to EM stack bounds)')
     buffersize= argschema.fields.Int(
         required=False, default=0, description='Buffer size to add')
+    LMstack_index = argschema.fields.Int(required=False, default=0,
+                                         description="index of LMstack list to create point matches between")
 
 class RenderModuleException(Exception):
     """Base Exception class for render module"""

renderapps/registration/find_principal_tile_overlaps.py

@@ -41,37 +41,51 @@ class CrossStackTilePairFile(argschema.schemas.mm.Schema):
     qstack = argschema.fields.Str(required=True)
     neighborPairs = argschema.fields.Nested(TilePair,many=True)
 
-def find_tile_pair(render,stack,ts,ref_stack):
- 
-    ts_geom = tilespec_to_bounding_box_polygon(ts)
-
-    width = ts.maxX-ts.minX
-    height = ts.maxY-ts.minY
-    minx = ts.minX
-    miny = ts.minY
-    p = {}
-    p['id']=ts.tileId
-    p['groupId']=ts.layout.sectionId
-
-    paired = render.run(renderapi.tilespec.get_tile_specs_from_box,ref_stack,ts.z,minx,miny,width,height)
+def find_principal_overlapping_tile(ts,search_ts,ts_geom=None):
+    if ts_geom is None:
+        ts_geom = tilespec_to_bounding_box_polygon(ts)
     overlap_tuples = []
-    for ts2 in paired:
+    for ts2 in search_ts:
         ts2_geom = tilespec_to_bounding_box_polygon(ts2)
         overlap = ts_geom.intersection(ts2_geom)
         frac_overlap = overlap.area/ts_geom.area
         overlap_tuples.append((ts2,frac_overlap))
     if len(overlap_tuples)==0:
-        raise RenderModuleException("tile {} in stack {} has no overlaps in stack paired={} {}".format(ts.tileId,stack,ref_stack,paired))
+        raise RenderModuleException("tile {} has no overlaps in {}".format(ts.tileId,paired))
     sorted_overlaps_tuples = sorted(overlap_tuples,key= lambda x: x[1])
     #print ts.tileId,sorted_overlaps_tuples[0][1],sorted_overlaps_tuples[-1][1]
     ts2 = sorted_overlaps_tuples[-1][0]
+    return ts2
+    
+def get_principal_overlapping_pair(ts,paired,ts_geom=None):
+    if ts_geom == None:
+        ts_geom = tilespec_to_bounding_box_polygon(ts)
+    p = {}
+    p['id']=ts.tileId
+    p['groupId']=ts.layout.sectionId
+    
+    ts2 = find_principal_overlapping_tile(ts,paired,ts_geom)
+    
     q = {}
     q['id']=ts2.tileId
     q['groupId']=ts2.layout.sectionId
     pair = {'p':p,'q':q}
     #print sorted_overlaps_tuples,ts2.tileId,ts.tileId
     return pair
 
+def find_tile_pair(render,stack,ts,ref_stack):
+ 
+    
+    ts_geom = tilespec_to_bounding_box_polygon(ts)
+    width = ts.maxX-ts.minX
+    height = ts.maxY-ts.minY
+    minx = ts.minX
+    miny = ts.minY
+
+    paired = render.run(renderapi.tilespec.get_tile_specs_from_box,ref_stack,ts.z,minx,miny,width,height)
+    
+    return get_principal_overlapping_pair(ts,paired,ts_geom=ts_geom)
+
 def create_principal_overlap_tile_pair(render,stack,ref_stack,pool_size=20,queryParameters={}):
     tilespecs = render.run(renderapi.tilespec.get_tile_specs_from_stack ,stack)
 

renderapps/registration/fit_transforms_by_point_match.py

@@ -15,10 +15,10 @@
         "project": "M246930_Scnn1a_4_f1",
         "client_scripts": "/pipeline/render/render-ws-java-client/src/main/scripts"
     },
-    "dst_stack": "FA_STI_DCV_FF_Session1",
-    "src_stack": "REG_STI_DCV_FF_Session3",
-    "output_stack": "FA_REG_STI_DCV_FF_Session3",
-    "matchcollection": "M246930_Scnn1a_4_f1_DAPI3_TO_DAPI1",
+    "dst_stack": "EMSite2_take2_EMA",
+    "src_stack": "Session1_DRP_STI_DCV_FF",
+    "output_stack": "TEST_Site2_take2_EMA_STI_DCV_FF_allSession_1",
+    "matchcollection": "M246930_Scnn1a_4_f1_DAPI1_EMsite2_ptMatch",
     "num_local_transforms": 0,
     "transform_type": "affine"
 }
@@ -41,22 +41,22 @@ class FitTransformsByPointMatchParameters(RenderParameters):
     num_local_transforms = Int(required=True,
                                description="number of local transforms to preserver, \
                                assumes point matches written down after such local transforms")
-    setz = Boolean(required=False, default = True,
-                               description="whether to change z's to the destination stack")
     transform_type = Str(required = False, default = 'affine',
                          validate = mm.validate.OneOf(["affine","rigid"]),
                          description = "type of transformation to fit")
     pool_size = Int(required=False, default=20,
                     description= 'degree of parallelism (default 20)')
-   
+    setz = Boolean(required=False, default = True,
+                   description="whether to change z's to the destination stack")
+    
 logger = logging.getLogger(__name__)
 
 def fit_transforms_by_pointmatch(render,
                                  src_stack,
                                  dst_stack,
                                  matchcollection,
                                  num_local_transforms,
-				 setz,
+                                 setz,
                                  Transform):
     print src_stack,dst_stack,matchcollection,num_local_transforms
     tilespecs_p = renderapi.tilespec.get_tile_specs_from_stack(src_stack, render=render)
@@ -66,29 +66,44 @@ def fit_transforms_by_pointmatch(render,
     for k,tsp in enumerate(tilespecs_p):
         pid=tsp.tileId
         pgroup = tsp.layout.sectionId
-        match = renderapi.pointmatch.get_matches_involving_tile(matchcollection,pgroup,pid,render=render)[0]
-        if match['qId']==pid:
-            pid = match['qId']
-            qid = match['pId']
-            p_pts = np.array(match['matches']['q']).T
-            q_pts = np.array(match['matches']['p']).T
-        else:
-            pid = match['pId']
-            qid = match['qId']
-            p_pts = np.array(match['matches']['p']).T
-            q_pts = np.array(match['matches']['q']).T
-
-        tsq = next(ts for ts in tilespecs_q if ts.tileId == qid)
-        tforms = tsq.tforms[num_local_transforms:]
-        dst_pts = renderapi.transform.estimate_dstpts(tforms,q_pts)
-        p_pts_global = renderapi.transform.estimate_dstpts(tsp.tforms[num_local_transforms:],p_pts)
-        final_tform = Transform()
-        final_tform.estimate(p_pts,dst_pts)
-        tsp.tforms=tsp.tforms[0:num_local_transforms]+[final_tform]
-        
-	if setz == True:
-		tsp.z = tsq.z
-
+        try:
+            matches = renderapi.pointmatch.get_matches_involving_tile(matchcollection,pgroup,pid,render=render)
+            dst_pts_list = []
+            p_pts_list = []
+            for match in matches:
+                if match['qId']==pid:
+                    pid = match['qId']
+                    qid = match['pId']
+                    p_pts = np.array(match['matches']['q']).T
+                    q_pts = np.array(match['matches']['p']).T
+                else:
+                    pid = match['pId']
+                    qid = match['qId']
+                    p_pts = np.array(match['matches']['p']).T
+                    q_pts = np.array(match['matches']['q']).T
+                try:
+                    tsq = next(ts for ts in tilespecs_q if ts.tileId == qid)
+                    tforms = tsq.tforms[num_local_transforms:]
+                    dst_pts = renderapi.transform.estimate_dstpts(tforms,q_pts)
+                    dst_pts_list.append(dst_pts)
+                    p_pts_list.append(p_pts)
+                except:
+                    pass
+                #p_pts_global = renderapi.transform.estimate_dstpts(tsp.tforms[num_local_transforms:],p_pts)
+            if len(dst_pts_list)>0:
+                dst_pts = np.vstack(dst_pts_list)
+                p_pts = np.vstack(p_pts_list)
+                final_tform = Transform()
+                final_tform.estimate(p_pts,dst_pts)
+                tsp.tforms=tsp.tforms[0:num_local_transforms]+[final_tform]
+                if setz == True:
+                    tsp.z = tsq.z
+                tilespecs_out.append(tsp)
+
+        except IndexError as e:
+            pass
+        except StopIteration as e:
+            pass
         # print pid,qid
         # print "p_pts"
         # print p_pts
@@ -98,8 +113,9 @@ def fit_transforms_by_pointmatch(render,
         # print p_pts_global
         # if k==1:
         #     break
+        
 
-    return tilespecs_p
+    return tilespecs_out
 
 
 class FitTransformsByPointMatch(RenderModule):
@@ -122,17 +138,18 @@ def run(self):
                                      self.args['matchcollection'],
                                      self.args['num_local_transforms'],
                                      self.args['setz'],
-				     Transform)
+                                     Transform)
 
         outstack = self.args['output_stack']
 
         self.render.run(renderapi.stack.delete_stack, outstack)
         self.render.run(renderapi.stack.create_stack, outstack)
-
+        sv=self.render.run(renderapi.stack.get_stack_metadata, self.args['dst_stack'])
+        
         renderapi.client.import_tilespecs_parallel(outstack,tilespecs,
                                                     render=self.render,
                                                     close_stack=True)
-        
+        self.render.run(renderapi.stack.set_stack_metadata,outstack, sv)
         self.render.run(renderapi.stack.set_stack_state,
                         outstack, state='COMPLETE')