Updated the repository with the newest versions of the NDH_Tools functions

Author: nholschuh

edgetrim_mask.py

@@ -0,0 +1,59 @@
+
+
+def edgetrim_mask(edge_trim_array, debug_flag=0, start_trim=0, end_trim=0):
+    """
+    % (C) Nick Holschuh - Amherst College -- 2022 (Nick.Holschuh@gmail.com)
+    %
+    % This function takes in a CSARP_surf y array generated using "pick edge_mask"
+    % and converts the output into a mask for subsetting CSARP_dem data
+    %
+    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+    % The inputs are:
+    %
+    %
+    %%%%%%%%%%%%%%%
+    % The outputs are:
+    %
+    %
+    %
+    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+    % the edge trim array is typically surf_data['surf']['y'][7]
+    """        
+    import numpy as np
+    from scipy.interpolate import interp1d
+    mask = np.zeros(edge_trim_array.shape)
+
+    et_row,et_col = np.where(~np.isnan(edge_trim_array))
+    center_split = np.mean(et_row)
+    left_ind = np.where(et_row < center_split)
+    right_ind = np.where(et_row >= center_split)
+
+
+    col_opts = np.arange(0,edge_trim_array.shape[1])
+
+    left_row_interpolator = interp1d(et_col[left_ind],et_row[left_ind],fill_value=np.NaN,bounds_error=0)
+    right_row_interpolator = interp1d(et_col[right_ind],et_row[right_ind],fill_value=np.NaN,bounds_error=0)
+
+    left_row = left_row_interpolator(col_opts).astype(int)
+    right_row = right_row_interpolator(col_opts).astype(int)
+
+    for i in col_opts:
+        mask[left_row[i]:right_row[i]+1,i] = 1
+    for i in range(start_trim):
+        mask[:,i] = 0
+    for i in range(end_trim):
+        mask[:,-i] = 0
+
+    mask[mask == 0] = np.NaN
+    
+    
+    if debug_flag:
+        import matplotlib.pyplot as pltdd
+        plt.imshow(mask)
+        plt.plot(et_col,et_row,'o',c='red')
+        plt.plot(col_opts,left_row,'-',c='black')
+        plt.plot(col_opts,right_row,'-',c='black')
+        plt.gca().set_aspect('auto')
+
+    return mask
+

find_nearest.py

@@ -27,6 +27,11 @@ def find_nearest(vector_2_search,value,how_many=1):
     %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     """
 
+    if isinstance(vector_2_search,list):
+        vector_2_search = np.array(vector_2_search)
+    if isinstance(value,list):
+        value = np.array(value)
+    
     comp_vec = vector_2_search
 
     if isinstance(vector_2_search,type(np.array([]))) == 0:

find_nearest_xy.py

@@ -27,13 +27,28 @@ def find_nearest_xy(vector_2_search,value,how_many=1):
     %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     """
 
+    if isinstance(vector_2_search,list):
+        vector_2_search = np.array(vector_2_search)
+    if isinstance(value,list):
+        value = np.array(value)
+
+        
     sd = vector_2_search.shape
-    
+    if sd[0] == 2:
+        vector_2_search = vector_2_search.transpose();
+    else:
+        vector_2_search = vector_2_search;
+
+    ss = value.shape
+    if len(ss) == 1:
+        value = np.expand_dims(value,axis=0)
+        ss = value.shape    
+
     ## The two column, vector search case
     if len(sd) == 2:
 
         comp_vec = vector_2_search[:,0] + vector_2_search[:,1]*np.sqrt(np.array(-1,dtype=complex));
-        ss = value.shape;
+        ss = value.shape
         if np.all([np.min(ss) > 1,ss[0] >= ss[1]]):
             value2 = value.transpose();
         elif ss[0] == 1:

find_pixelcoords.py

@@ -26,7 +26,7 @@ def find_pixelcoords(im_filename,original_width,original_height,im_pick_params=0
     %                             ### ---- minimum contour size (number of points)
     %                             ### ---- the aggregation type (0: average of marker, 1: horizontal bar)
     %                             ### ---- distance threshold for pixel combination or separation
-    %                             ### ---- the distance calculation method (0:true or 1:vertical)
+    %                             ### ---- the distance calculation method (0:true or 1:vertical or 2:overweight horizontal)
     %                       0 defaults to [[0,5,0,5],[2,25,1,10]] which looks for blue points and red lines
     %
     %%%%%%%%%%%%%%%
@@ -71,7 +71,7 @@ def find_pixelcoords(im_filename,original_width,original_height,im_pick_params=0
         im_colors = ['blue','red']
         im_pick_params = [[0,5,0,50,2],[2,25,1,10,1]] 
         ### This defines how the contours are treated. There should be three parameters here:
-                                 ### ---- which bands to look for minima in
+                                 ### ---- which bands to look for minima in (0: red, 1: green, 2:blue)
                                  ### ---- minimum contour size (number of points)
                                  ### ---- the aggregation type (0: average of marker, 1: horizontal bar)
                                  ### ---- distance threshold for pixel combination or separation

flatten_list.py

@@ -1,7 +1,7 @@
 import sys
 sys.path.append('/mnt/data01/Code/')
 
-def flatten_list(list_of_lists,recursive=1):
+def flatten_list(list_of_lists,recursive_flag=1,nan_divide=0):
     """
     % (C) Nick Holschuh - Amherst College - 2022 (Nick.Holschuh@gmail.com)
     % This will take a list of lists (or an array of arrays) and flattens
@@ -10,7 +10,7 @@ def flatten_list(list_of_lists,recursive=1):
     % The inputs are as follows:
     %
     % flatten_list -- The list of lists to be flattened
-    % recursive -- This will keep flattening until the list stops changing
+    % recursive_flag -- This will keep flattening until the list stops changing
     % 
     %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     % The outputs are as follows:
@@ -24,6 +24,14 @@ def flatten_list(list_of_lists,recursive=1):
     import numpy as np
     from NDH_Tools import compare_list
 
+    if nan_divide == 1:
+        new_list_of_lists = []
+        for i in list_of_lists:
+            new_list_of_lists.append(i)
+            new_list_of_lists.append([np.nan])
+        list_of_lists = new_list_of_lists[:-1]
+    
+    
     flat_list = []
     array_flag = 0
     steps = 0
@@ -49,7 +57,7 @@ def flatten_list(list_of_lists,recursive=1):
 
         if comb_result:
             recursive_flag = 0
-        if recursive == 0:
+        if recursive_flag == 0:
             recursive_flag = 0
 
         ############ nan's present in the ojbect break the list comparison.

generate_pickingpdf.py

@@ -9,7 +9,7 @@
 ################################################################################################
 
 
-def generate_pickingpdf(fn,picking_root_dir,frame_spacing=25,surf_dir='CSARP_surf_ndh',crop_type='100'):
+def generate_pickingpdf(fn,picking_root_dir,frame_spacing=25,surf_dir='CSARP_surf_ndh',crop_type='100',clims=[], alternative_data_opt=0):
     """
     % (C) Nick Holschuh - Amherst College -- 2022 (Nick.Holschuh@gmail.com)
     %
@@ -115,60 +115,102 @@ def generate_pickingpdf(fn,picking_root_dir,frame_spacing=25,surf_dir='CSARP_sur
 
         os.system('rm -r '+picking_root_dir+seg)
 
-        
+#################################################################################################################### 
+#################################################################################################################### 
+#################################################################################################################### 
+####################################################################################################################  
+####################################################################################################################        
     #######################################################################################################
     ####################### The alternative is a directory full of standard files
     str_opt, str_ind = ndh.str_compare([fn],'standard')
     if len(str_opt) > 0:    
+        
         file_list = sorted(glob.glob(fn+'/Data_*.mat'))
+        
+        if len(file_list) > 0:
+            ki = [];
+            for ind,i in enumerate(file_list):
+                if i.split('/')[-1][5] != 'i':
+                    ki.append(ind)
+            file_list = ndh.index_list(file_list,ki)
+        else:
+            file_list = sorted(glob.glob(fn+'/*.mat'))
 
-        max_num = int(file_list[-1].split('/')[-1].split('.')[0].split('_')[-1])
-        seg = file_list[-1].split('/')[-2]
-        froot = '_'.join(file_list[-1].split('/')[-1].split('.')[0].split('_')[0:-1])
+            
+        try:
+            max_num = int(file_list[-1].split('/')[-1].split('.')[0].split('_')[-1])
+            seg = file_list[-1].split('/')[-2]
+            froot = '_'.join(file_list[-1].split('/')[-1].split('.')[0].split('_')[0:-1])
+        except:
+            max_num = len(file_list)
+            seg = file_list[-1].split('/')[-2]
+            froot = seg
         num_range = np.arange(1,max_num+1,1)
 
+        
 
         ################ Here we do some directory checking to make sure all folders we need exist
         if not os.path.isdir(picking_root_dir+'To_Pick/'+seg):
             os.makedirs(picking_root_dir+'To_Pick/'+seg)     ### The pdf directory where pickfiles are stored 
 
-        if not os.path.isdir(picking_root_dir+seg):
-            os.makedirs(picking_root_dir+seg)                ### The directory for the segment
-
         fig = plt.figure(figsize=(15,7))
         ax = plt.gca()  
 
         for ind1,file_num in enumerate(num_range):
+            
             fn_frame = '%s/%s_%0.3d.mat' % (fn,froot,file_num)
+            print(fn_frame)
+            if os.path.isfile(fn_frame) == 0:
+                fn_frame = file_list[ind1]
+
             radar_data,depth_data = ndh.radar_load(fn_frame,plot_flag=0,elevation1_or_depth2=0)
 
-            bot_inds = ndh.find_nearest(radar_data['Time'],radar_data['Bottom'])['index'].astype(float)
-            surf_inds = ndh.find_nearest(radar_data['Time'],radar_data['Surface'])['index'].astype(float)        
-            bot_inds[bot_inds == 0] = np.NaN
-            surf_inds[surf_inds == 0] = np.NaN
-            
-            
-            if crop_type == '100':
-                bot_ind = np.nanmax(bot_inds)+100
-                crop_string = 'maxbotplus100'
+            if len(radar_data.keys()) > 0:
+                bot_inds = ndh.find_nearest(radar_data['Time'],radar_data['Bottom'])['index'].astype(float)
+                surf_inds = ndh.find_nearest(radar_data['Time'],radar_data['Surface'])['index'].astype(float)        
+                bot_inds[bot_inds == 0] = np.NaN
+                surf_inds[surf_inds == 0] = np.NaN
+
+
+                if crop_type == '100':
+                    bot_ind = np.nanmax(bot_inds)+100
+                    crop_string = 'maxbotplus100'
+                else:
+                    bot_ind = len(radar_data['Time'])
+                    crop_string = 'nocrop'
+                    
+                ndh.remove_image(ax,1,verbose=0)
+                ndh.remove_line(ax,2,verbose=0)
+
+                if np.isnan(bot_ind) == 1:
+                    plt.plot(0,0)
+                else:
+                    ########### This accomodates files that have more than one data type
+                    if alternative_data_opt == 1:
+                        find_data_opts = ndh.str_compare(radar_data.keys,'Data2')
+                        if len(find_data_opts[0]) > 0:
+                            radar_data['Data'] = radar_data['Data2']
+
+                    if len(clims) > 0:
+                        imdata = plt.imshow(10*np.log10(radar_data['Data'][:int(bot_ind),:]),
+                                            origin='lower',aspect='auto',cmap='gray_r',vmin=clims[0],vmax=clims[1])
+                    else:
+                        imdata = plt.imshow(10*np.log10(radar_data['Data'][:int(bot_ind),:]),
+                                            origin='lower',aspect='auto',cmap='gray_r')                
+
+                    plt.plot(np.arange(0,len(radar_data['distance'])),bot_inds,ls='--',c='green',alpha=0.2)
+                    plt.plot(np.arange(0,len(radar_data['distance'])),surf_inds,ls='--',c='green',alpha=0.2)
+                
             else:
-                bot_ind = len(radar_data['Time'])
-                crop_string = 'nocrop'
+                plt.plot(0,0)
 
-            ndh.remove_image(ax,1,verbose=0)
-            ndh.remove_line(ax,2,verbose=0)
-
-            imdata = plt.imshow(np.log10(radar_data['Data'][:int(bot_ind),:]),
-                                origin='lower',aspect='auto',cmap='gray_r')
-
-            plt.plot(np.arange(0,len(radar_data['distance'])),bot_inds,ls='--',c='green',alpha=0.2)
-            plt.plot(np.arange(0,len(radar_data['distance'])),surf_inds,ls='--',c='green',alpha=0.2)
 
             ax.invert_yaxis()
 
             plt.axis('off')
-            plt.savefig('%s%s/Frame_%0.3d.png' %(picking_root_dir,seg,file_num))
-
+            frame_fn = '%s/To_Pick/%s/Frame_%0.3d.png' %(picking_root_dir,seg,file_num)
+            plt.savefig(frame_fn)
+            print('  --- Completed Image '+str(ind1)+' of '+str(len(num_range)))
 
         print('Completed the image generation')
 
@@ -180,9 +222,14 @@ def generate_pickingpdf(fn,picking_root_dir,frame_spacing=25,surf_dir='CSARP_sur
         pdfend = 'StandardPicks_%s_crop_%s.pdf' %(seg,crop_string)
         pdfname=pdfroot+pdfend
 
-        frames = '%s%s/*.png' % (picking_root_dir,seg)
+        frames = '%s/To_Pick/%s/*.png' % (picking_root_dir,seg)
 
         os.system("convert -adjoin "+frames+" -gravity center -scale '90<x770<' "+pdfname)
         ###########
 
-        os.system('rm -r '+picking_root_dir+seg)
+        if 0:
+            os.system('rm -r '+picking_root_dir+'/To_Pick/'+seg)
+            os.system('rmdir '+picking_root_dir+'/To_Pick/'+seg)
+        else:
+            print('rm -r '+picking_root_dir+'/To_Pick/'+seg)
+            print('rmdir '+picking_root_dir+'/To_Pick/'+seg)           

get_contour_data.py

@@ -1,7 +1,18 @@
+################ This is the import statement required to reference scripts within the package
+import os,sys
+ndh_tools_path_opts = [
+    '/mnt/data01/Code/',
+    '/home/common/HolschuhLab/Code/'
+]
+for i in ndh_tools_path_opts:
+    if os.path.isfile(i): sys.path.append(i)
+################################################################################################
+
 import numpy as np
 import matplotlib.pyplot as plt
+import NDH_Tools as ndh
 
-def get_contour_data(contour_object):
+def get_contour_data(contour_object,simplify_flag=0,simplify_threshold=100):
     """
     % (C) Nick Holschuh - Amherst College - 2022 (Nick.Holschuh@gmail.com)
     % 
@@ -27,5 +38,14 @@ def get_contour_data(contour_object):
                 contour_mat = np.array(j.vertices)
                 contour_mat = np.concatenate([contour_mat,np.ones((len(contour_mat[:,0]),1))*cvals[ind0]],axis=1)
                 clines.append(contour_mat)
+
+    if simplify_flag == 1:
+        new_clines = np.empty(shape=(0,3))
+        for ind0,subline in enumerate(clines):
+            if len(subline) > simplify_threshold:
+                new_clines = np.concatenate([new_clines,np.ones([1,3])*np.NaN,np.array(subline)])
+    else:
+        new_clines = clines 
+        
 
-    return clines
+    return new_clines