Skip to content

Commit a3c1fa8

Browse files
adamltayloradamltaylor
committed
Added run_some_tiles_through_1234_stages.m
1 parent 3e75d9a commit a3c1fa8

3 files changed

+181
-6
lines changed

‎invert_map_array.m

+9-4
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,16 @@
1-
function a_from_b = invert_map_array(b_from_a)
1+
function a_from_b = invert_map_array(b_from_a, max_b)
22
% b_from_a a vector, each element an integer >= 1.
33
% On return, a_from_b is such that a_from_b(b_from_a(i)) == i for all integer i
4-
% on [1, n], where n is length(b_from_a).
4+
% on [1, n], where n is length(b_from_a).
55
%
6-
% If b_from_a represents a function, a_from_b represents it's inverse.
6+
% If b_from_a represents a function, a_from_b represents its inverse.
7+
%
8+
% If max_b is given and is nonempty, a_from_b will be of length max_b. Otherwise a_from_b will
9+
% be of length max(b_from_a).
10+
if ~exist('max_b', 'var') || isempty(max_b) ,
11+
max_b = max(b_from_a) ;
12+
end
713
max_a = length(b_from_a) ;
8-
max_b = max(b_from_a) ;
914
if iscolumn(b_from_a) ,
1015
a_from_b = zeros(max_b, 1) ;
1116
else

‎run_all_tiles_through_1234_stages.m

+2-2
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@
1717
do_force_computation) ;
1818
tile_index_from_tile_ijk1 = raw_tile_index.tile_index_from_tile_ijk1 ;
1919
ijk1_from_tile_index = raw_tile_index.ijk1_from_tile_index ;
20-
xyz_from_tile_index = raw_tile_index.xyz_from_tile_index ;
20+
%xyz_from_tile_index = raw_tile_index.xyz_from_tile_index ;
2121
relative_path_from_tile_index = raw_tile_index.relative_path_from_tile_index ;
22-
raw_tile_map_shape = size(tile_index_from_tile_ijk1)
22+
%raw_tile_map_shape = size(tile_index_from_tile_ijk1)
2323
tile_count = length(relative_path_from_tile_index)
2424

2525

‎run_some_tiles_through_1234_stages.m

+170
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
% Set up the par pool
2+
use_this_fraction_of_cores(1) ;
3+
4+
% Set params for early jobs
5+
do_use_bsub = true ;
6+
do_actually_submit = true ;
7+
max_running_slot_count = inf ;
8+
bsub_option_string = '-P mouselight -J mouselight-pipeline-123' ;
9+
slots_per_job = 4 ;
10+
stdouterr_file_path = '' ; % will go to /dev/null
11+
12+
13+
% Build the full tile index
14+
raw_tile_index = compute_or_read_from_memo(sample_memo_folder_path, ...
15+
'raw-tile-index', ...
16+
@()(build_raw_tile_index(raw_root_path)), ...
17+
do_force_computation) ;
18+
full_tile_index_from_full_tile_ijk1 = raw_tile_index.tile_index_from_tile_ijk1 ;
19+
full_tile_ijk1_from_full_tile_index = raw_tile_index.ijk1_from_tile_index ;
20+
xyz_from_full_tile_index = raw_tile_index.xyz_from_tile_index ;
21+
relative_path_from_full_tile_index = raw_tile_index.relative_path_from_tile_index ;
22+
full_tile_grid_shape = size(full_tile_index_from_full_tile_ijk1)
23+
full_tile_count = length(relative_path_from_full_tile_index)
24+
25+
26+
% Get out just the tiles around the center_tile
27+
[tile_index_from_tile_ijk1, ...
28+
tile_ijk1_from_tile_index, ...
29+
xyz_from_tile_index, ...
30+
relative_path_from_tile_index] = ...
31+
extract_tiles_near_tile(full_tile_index_from_full_tile_ijk1, ...
32+
full_tile_ijk1_from_full_tile_index, ...
33+
xyz_from_full_tile_index, ...
34+
relative_path_from_full_tile_index, ...
35+
center_tile_relative_path) ;
36+
tile_count = length(relative_path_from_tile_index)
37+
38+
%
39+
% Figure out which tiles need to be run
40+
%
41+
42+
%is_to_be_run_from_tile_index = false(tile_count, 1) ;
43+
%is_to_be_run_from_tile_index(1:20) = true ;
44+
fprintf('Determining which of the %d tiles need to be LPLed...\n', tile_count) ;
45+
if do_force_computation ,
46+
is_to_be_run_from_tile_index = true(tile_count, 1) ;
47+
else
48+
is_to_be_run_from_tile_index = true(tile_count,1) ;
49+
parfor_progress(tile_count) ;
50+
parfor tile_index = 1 : tile_count
51+
tile_relative_path = relative_path_from_tile_index{tile_index} ;
52+
channel_0_tile_file_name = landmark_file_name_from_tile_relative_path(tile_relative_path, 0) ;
53+
channel_0_tile_file_path = fullfile(landmark_root_path, tile_relative_path, channel_0_tile_file_name) ;
54+
channel_1_tile_file_name = landmark_file_name_from_tile_relative_path(tile_relative_path, 1) ;
55+
channel_1_tile_file_path = fullfile(landmark_root_path, tile_relative_path, channel_1_tile_file_name) ;
56+
is_to_be_run = ...
57+
~(exist(channel_0_tile_file_path, 'file') && exist(channel_1_tile_file_path, 'file')) ;
58+
is_to_be_run_from_tile_index(tile_index) = is_to_be_run ;
59+
parfor_progress() ;
60+
end
61+
parfor_progress(0) ;
62+
end
63+
fprintf('Done determining which of the %d tiles need to be LPLed.\n', tile_count) ;
64+
tile_index_from_tile_to_be_run_index = find(is_to_be_run_from_tile_index) ;
65+
relative_path_from_tile_to_be_run_index = relative_path_from_tile_index(is_to_be_run_from_tile_index) ;
66+
tile_to_be_run_count = length(tile_index_from_tile_to_be_run_index)
67+
68+
69+
%
70+
% Run stages 123 on all tiles
71+
%
72+
73+
% Create the bqueue
74+
if do_use_bsub ,
75+
fprintf('Queueing LPL on %d tiles...\n', tile_to_be_run_count) ;
76+
bqueue = bqueue_type(do_actually_submit, max_running_slot_count) ;
77+
parfor_progress(tile_to_be_run_count) ;
78+
for tile_to_be_run_index = 1 : tile_to_be_run_count ,
79+
tile_relative_path = relative_path_from_tile_to_be_run_index{tile_to_be_run_index} ;
80+
%output_folder_path = fullfile(line_fix_path, tile_relative_path) ;
81+
%ensure_folder_exists(output_folder_path) ; % so stdout has somewhere to go
82+
%stdouterr_file_path = fullfile(output_folder_path, 'stdouterr-run-2.txt') ;
83+
bqueue.enqueue(slots_per_job, stdouterr_file_path, bsub_option_string, ...
84+
@lpl_process_single_tile, ...
85+
tile_relative_path, ...
86+
raw_root_path, ...
87+
line_fix_root_path, ...
88+
p_map_root_path, ...
89+
landmark_root_path, ...
90+
do_line_fix, ...
91+
do_force_computation) ;
92+
parfor_progress() ;
93+
end
94+
parfor_progress(0) ;
95+
fprintf('Done queueing LPL on %d tiles.\n\n', tile_to_be_run_count) ;
96+
97+
fprintf('LPLing queue on %d tiles...\n', length(bqueue.job_ids)) ;
98+
maximum_wait_time = inf ;
99+
do_show_progress_bar = true ;
100+
tic_id = tic() ;
101+
job_statuses = bqueue.run(maximum_wait_time, do_show_progress_bar) ;
102+
toc(tic_id)
103+
fprintf('Done LPLing queue on %d tiles.\n', length(bqueue.job_ids)) ;
104+
job_statuses
105+
successful_job_count = sum(job_statuses==1)
106+
else
107+
% Useful for debugging
108+
parfor_progress(tile_to_be_run_count) ;
109+
for tile_to_be_run_index = 1 : tile_to_be_run_count ,
110+
tile_relative_path = relative_path_from_tile_to_be_run_index{tile_to_be_run_index} ;
111+
%output_folder_path = fullfile(line_fix_path, tile_relative_path) ;
112+
%ensure_folder_exists(output_folder_path) ; % so stdout has somewhere to go
113+
%stdouterr_file_path = fullfile(output_folder_path, 'stdouterr-run-2.txt') ;
114+
lpl_process_single_tile( ...
115+
tile_relative_path, ...
116+
raw_root_path, ...
117+
line_fix_root_path, ...
118+
p_map_root_path, ...
119+
landmark_root_path, ...
120+
do_line_fix, ...
121+
do_force_computation) ;
122+
parfor_progress() ;
123+
end
124+
parfor_progress(0) ;
125+
end
126+
127+
128+
%
129+
% Run stage 4 (landmark matching)
130+
% We bqueue up a single job that will use 48 cores
131+
%
132+
133+
% New params for bjobs
134+
do_use_bsub = true ;
135+
do_actually_submit = true ;
136+
max_running_slot_count = inf ;
137+
bsub_option_string = '-P mouselight -J mouselight-landmark-matching' ;
138+
slots_per_job = 48 ;
139+
stdouterr_file_path = fullfile(script_folder_path, 'mouselight-landmark-matching.out.txt') ;
140+
141+
% Create the bqueue
142+
if do_use_bsub ,
143+
fprintf('Running landmark-matching (as single bsub job)...\n') ;
144+
bqueue = bqueue_type(do_actually_submit, max_running_slot_count) ;
145+
bqueue.enqueue(slots_per_job, stdouterr_file_path, bsub_option_string, ...
146+
@landmark_match_all_tiles_in_z, ...
147+
raw_root_path, ...
148+
sample_memo_folder_path, ...
149+
landmark_root_path, ...
150+
z_point_match_root_path, ...
151+
do_force_computation) ;
152+
maximum_wait_time = inf ;
153+
do_show_progress_bar = false ;
154+
tic_id = tic() ;
155+
job_statuses = bqueue.run(maximum_wait_time, do_show_progress_bar) ;
156+
toc(tic_id)
157+
if all(job_statuses==1) ,
158+
fprintf('Done running landmark-matching.\n\n') ;
159+
else
160+
fprintf('Done running landmark-matching, but the job returned a non-zero error code.\n\n') ;
161+
end
162+
else
163+
% Useful for debugging
164+
landmark_match_all_tiles_in_z( ...
165+
raw_root_path, ...
166+
sample_memo_folder_path, ...
167+
landmark_root_path, ...
168+
z_point_match_root_path, ...
169+
do_force_computation) ;
170+
end

0 commit comments

Comments
 (0)