|
| 1 | +import argparse |
| 2 | +import json |
| 3 | +import os |
| 4 | +import pandas as pd |
| 5 | +import subprocess |
| 6 | +import copy |
| 7 | +import renderapi |
| 8 | +from renderapi.tilespec import TileSpec,Layout,MipMapLevel |
| 9 | +from renderapi.transform import AffineModel |
| 10 | +from create_mipmaps import create_mipmaps |
| 11 | +my_env = os.environ.copy() |
| 12 | +from itertools import izip_longest |
| 13 | +from pathos.multiprocessing import Pool |
| 14 | +from render_module import RenderModule,RenderParameters |
| 15 | +from json_module import InputFile,InputDir |
| 16 | +import marshmallow as mm |
| 17 | + |
| 18 | +class CreateFastStacksParameters(RenderParameters): |
| 19 | + statetableFile = InputFile(required=True, |
| 20 | + metadata={'description':'state table file'}) |
| 21 | + projectDirectory = InputDir(required=True, |
| 22 | + metadata={'description':'path to project root'}) |
| 23 | + outputStackPrefix = mm.fields.Str(required=False,default="ACQ", |
| 24 | + metadata={'description':'prefix to include in front of channel name for render stack'}) |
| 25 | + pool_size = mm.fields.Int(require=False,default=20, |
| 26 | + metadata={'description':'number of parallel threads to use'}) |
| 27 | + |
| 28 | +def make_tilespec_from_statetable (df,rootdir,outputProject,outputOwner,outputStack,minval=0,maxval=50000): |
| 29 | + df = df[df['zstack']==0] |
| 30 | + #ribbons = df.groupby('ribbon') |
| 31 | + #zoffset=0 |
| 32 | + |
| 33 | + #for ribbnum,ribbon in ribbons: |
| 34 | + # ribbon.loc[ribbon.index,'z']=ribbon['section']+zoffset |
| 35 | + # zoffset += ribbon['section'].max()+1 |
| 36 | + # df.loc[ribbon.index,'z']=ribbon['z'].values |
| 37 | + |
| 38 | + |
| 39 | + mipmap_args = [] |
| 40 | + tilespecpaths = [] |
| 41 | + for ((ch,sess),chgroup) in df.groupby(['ch_name','session']): |
| 42 | + print ch,sess |
| 43 | + |
| 44 | + for ((rib,sect),group) in chgroup.groupby(['ribbon','section']): |
| 45 | + tilespeclist=[] |
| 46 | + z=0 |
| 47 | + for ind,row in group.iterrows(): |
| 48 | + filepath=row.full_path |
| 49 | + fileparts=filepath.split(os.path.sep)[1:] |
| 50 | + tilespecdir = rootdir + "/processed/downsamp_tilespec/"+fileparts[5]+"/"+fileparts[6]+"/"+fileparts[7] |
| 51 | + #print tilespecdir |
| 52 | + if not os.path.isdir(tilespecdir): |
| 53 | + os.makedirs(tilespecdir) |
| 54 | + downdir = rootdir+"/processed/downsamp_images/"+fileparts[5]+"/"+fileparts[6]+"/"+fileparts[7] |
| 55 | + #print "This is the Down Sampled Directory: %s"%downdir |
| 56 | + |
| 57 | + if not os.path.exists(downdir): |
| 58 | + os.makedirs(downdir) |
| 59 | + |
| 60 | + #construct command for creating mipmaps for this tilespec |
| 61 | + #downcmd = ['python','create_mipmaps.py','--inputImage',filepath,'--outputDirectory',downdir,'--mipmaplevels','1','2','3'] |
| 62 | + #cmds.append(downcmd) |
| 63 | + mipmap_args.append((filepath,downdir)) |
| 64 | + layout = Layout(sectionId=row.ribbon*1000+row.section, |
| 65 | + scopeId='Leica', |
| 66 | + cameraId='zyla', |
| 67 | + imageRow=0, |
| 68 | + imageCol=0, |
| 69 | + stageX = row.xstage, |
| 70 | + stageY = row.ystage, |
| 71 | + rotation = 0.0, |
| 72 | + pixelsize = row.scale_x) |
| 73 | + |
| 74 | + mipmap0 = MipMapLevel(level=0,imageUrl=row.full_path) |
| 75 | + mipmaplevels=[mipmap0] |
| 76 | + filename = "%s_S%04d_F%04d_Z%02d.tif"%(row.ch_name,row.section,row.frame,0) |
| 77 | + for i in range(1,4): |
| 78 | + scUrl = 'file:' + os.path.join(downdir,filename[0:-4]+'_mip0%d.jpg'%i) |
| 79 | + mml = MipMapLevel(level=i,imageUrl=scUrl) |
| 80 | + mipmaplevels.append(mml) |
| 81 | + |
| 82 | + tform = AffineModel(M00=row.a00, |
| 83 | + M01=row.a01, |
| 84 | + M10=row.a10, |
| 85 | + M11=row.a11, |
| 86 | + B0=row.a02, |
| 87 | + B1=row.a12) |
| 88 | + |
| 89 | + tilespeclist.append(TileSpec(tileId=row.tileID, |
| 90 | + frameId = row.frame, |
| 91 | + z=row.z, |
| 92 | + width=row.width, |
| 93 | + height=row.height, |
| 94 | + mipMapLevels=mipmaplevels, |
| 95 | + tforms=[tform], |
| 96 | + minint=minval, |
| 97 | + maxint=maxval, |
| 98 | + layout= layout)) |
| 99 | + z = row.z |
| 100 | + |
| 101 | + json_text=json.dumps([t.to_dict() for t in tilespeclist],indent=4) |
| 102 | + json_file = os.path.join(tilespecdir,outputProject+'_'+outputOwner+'_'+outputStack+'_%04d.json'%z) |
| 103 | + fd=open(json_file, "w") |
| 104 | + fd.write(json_text) |
| 105 | + fd.close() |
| 106 | + tilespecpaths.append(json_file) |
| 107 | + return tilespecpaths,mipmap_args |
| 108 | + |
| 109 | +def create_mipmap_from_tuple(mipmap_tuple): |
| 110 | + (filepath,downdir)=mipmap_tuple |
| 111 | + return create_mipmaps(filepath,downdir) |
| 112 | + |
| 113 | +class CreateFastStack(RenderModule): |
| 114 | + def __init__(self,schema_type=None,*args,**kwargs): |
| 115 | + if schema_type is None: |
| 116 | + schema_type = CreateFastStacksParameters |
| 117 | + |
| 118 | + super(CreateFastStack,self).__init__(schema_type=schema_type,*args,**kwargs) |
| 119 | + def run(self): |
| 120 | + outputProject=self.args['render']['project'] |
| 121 | + outputOwner = self.args['render']['owner'] |
| 122 | + statetablefile = self.args['statetableFile'] |
| 123 | + rootdir = self.args['projectDirectory'] |
| 124 | + |
| 125 | + df = pd.read_csv(statetablefile) |
| 126 | + ribbons = df.groupby('ribbon') |
| 127 | + k=0 |
| 128 | + pool = Pool(self.args['pool_size']) |
| 129 | + for ribnum,ribbon in ribbons: |
| 130 | + mydf = ribbon.groupby('ch_name') |
| 131 | + for channum,chan in mydf: |
| 132 | + outputStack = self.args['outputStackPrefix'] + '_%s'%(channum) |
| 133 | + |
| 134 | + self.logger.info("creating tilespecs and cmds....") |
| 135 | + tilespecpaths,mipmap_args = make_tilespec_from_statetable(chan,rootdir,outputProject,outputOwner,outputStack,0,65000) |
| 136 | + self.logger.info("importing tilespecs into render....") |
| 137 | + self.logger.info("creating downsampled images ...") |
| 138 | + |
| 139 | + results=pool.map(create_mipmap_from_tuple,mipmap_args) |
| 140 | + |
| 141 | + #groups = [(subprocess.Popen(cmd,\ |
| 142 | + # stdout=subprocess.PIPE) for cmd in cmds)] \ |
| 143 | + # * self.args['pool_size'] # itertools' grouper recipe |
| 144 | + #for processes in izip_longest(*groups): # run len(processes) == limit at a time |
| 145 | + # for p in filter(None, processes): |
| 146 | + # p.wait() |
| 147 | + self.logger.info("uploading to render ...") |
| 148 | + if k==0: |
| 149 | + #renderapi.stack.delete_stack(outputStack,owner=outputOwner, |
| 150 | + #project=outputProject,render=self.render) |
| 151 | + renderapi.stack.create_stack(outputStack,owner=outputOwner, cycleNumber=1, cycleStepNumber=1, |
| 152 | + project=outputProject,verbose=False,render=self.render) |
| 153 | + print k |
| 154 | + self.logger.info(tilespecpaths) |
| 155 | + renderapi.client.import_jsonfiles_parallel(outputStack,tilespecpaths,render=self.render) |
| 156 | + |
| 157 | + k+=1 |
| 158 | + |
| 159 | +if __name__ == "__main__": |
| 160 | + |
| 161 | + mod = CreateFastStack(schema_type = CreateFastStacksParameters) |
| 162 | + |
| 163 | + mod.run() |
| 164 | + |
0 commit comments