Skip to content

Commit 5288524

Browse files
adamltayloradamltaylor
committed
master: Added example sample-processing folder sample-2022-02-01, to illustrate approximate usage
1 parent 841646a commit 5288524

27 files changed

+10283
-0
lines changed

‎sample-2022-02-01/README.txt

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
This is example code for running a sample through the Mouselight
2+
pipeline. It was designed to work with commit db1b0a0 of the
3+
mouselight_toolbox, from 2022-03-03, but hopefully would work with
4+
minor modifications on the current commit of mouselight_toolbox.
5+
Difficult to test b/c all this data has been compressed and put into
6+
cold storage.
7+
8+
ALT
9+
2024-04-04

‎sample-2022-02-01/check-tiles-2022-02-01-after-fixing-broken-tiles-run-01.out.txt

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
>> cclf; check_tiles('2022-02-01', true)
2+
3+
raw_tile_map_shape =
4+
5+
29 18 58
6+
7+
Raw tile folder count: 15895
8+
Nominal tile shape (yxz) is: [1536 1024 251]
9+
Nominal raw tiff file size is: 789618398
10+
Nominal line-fix tiff file size is: 789618398
11+
12+
Checking stage raw...
13+
[**************************************************]: 100.0%
14+
15+
Problematic raw file count: 0
16+
17+
Checking stage line-fix...
18+
[**************************************************]: 100.0%
19+
20+
Problematic line-fix file count: 0
21+
22+
Checking stage classifier...
23+
[**************************************************]: 100.0%
24+
25+
Problematic classifier file count: 0
26+
27+
Checking stage descriptor...
28+
[**************************************************]: 100.0%
29+
30+
Problematic descriptor file count: 0
31+
32+
Checking stage point-match...
33+
[**************************************************]: 100.0%
34+
35+
Problematic point-match file count: 0
36+
>>

‎sample-2022-02-01/check-tiles-2022-02-01-run-01.out.txt

+1,127
Large diffs are not rendered by default.

‎sample-2022-02-01/check_once_broken_tiles.m

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
sample_date = '2022-02-01' ;
2+
broken_tile_paths = { ...
3+
'2022-02-05/02/02572', ...
4+
'2022-02-05/02/02638', ...
5+
'2022-02-05/02/02646', ...
6+
'2022-02-05/02/02650', ...
7+
'2022-02-05/02/02664', ...
8+
'2022-02-05/02/02668', ...
9+
'2022-02-05/02/02672', ...
10+
'2022-02-05/02/02686', ...
11+
'2022-02-05/02/02690', ...
12+
'2022-02-05/02/02698', ...
13+
'2022-02-05/02/02699', ...
14+
'2022-02-05/02/02704', ...
15+
'2022-02-05/02/02713', ...
16+
'2022-02-05/02/02730', ...
17+
'2022-02-05/02/02734' } ;
18+
19+
do_show_progress = true ;
20+
check_some_tiles(sample_date, broken_tile_paths, do_show_progress) ;

‎sample-2022-02-01/compare_in_place_line_fixed_to_production_2022_02_01.m

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
% Set parameters
2+
do_force_computation = false ;
3+
4+
% Build an index of the paths to raw tiles
5+
sample_date = '2022-02-01' ;
6+
script_folder_path = fileparts(mfilename('fullpath')) ;
7+
fluorescence_root_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s/Tiling', sample_date) ;
8+
sample_memo_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/cluster/Reconstructions/%s', sample_date) ;
9+
line_fix_root_path = sprintf('/nrs/mouselight/pipeline_output/%s/stage_1_line_fix_output', sample_date) ;
10+
11+
12+
% Set up the par pool
13+
use_this_fraction_of_cores(1) ;
14+
15+
% Build the tile index
16+
raw_tile_index = compute_or_read_from_memo(sample_memo_folder_path, ...
17+
'raw-tile-index', ...
18+
@()(build_raw_tile_index(fluorescence_root_path)), ...
19+
do_force_computation) ;
20+
tile_index_from_tile_ijk1 = raw_tile_index.tile_index_from_tile_ijk1 ;
21+
ijk1_from_tile_index = raw_tile_index.ijk1_from_tile_index ;
22+
%xyz_from_tile_index = raw_tile_index.xyz_from_tile_index ;
23+
relative_path_from_tile_index = raw_tile_index.relative_path_from_tile_index ;
24+
%raw_tile_map_shape = size(tile_index_from_tile_ijk1)
25+
tile_count = length(relative_path_from_tile_index) %#ok<NOPTS>
26+
27+
28+
do_shifts_match_from_tile_index = false(tile_count,1) ;
29+
pbo = progress_bar_object(tile_count) ;
30+
parfor tile_index = 1 : tile_count ,
31+
tile_relative_path = relative_path_from_tile_index{tile_index} ;
32+
xlineshift_file_path = fullfile(line_fix_root_path, tile_relative_path, 'Xlineshift.txt') ;
33+
production_shift = read_xlineshift_file(xlineshift_file_path) ;
34+
fluorescence_tile_path = fullfile(fluorescence_root_path, tile_relative_path) ;
35+
tile_metadata = read_tile_metadata(fluorescence_tile_path) ;
36+
in_place_shift = tile_metadata.shift ;
37+
do_shifts_match_from_tile_index(tile_index) = (production_shift==in_place_shift) ;
38+
pbo.update() ; %#ok<PFBNS>
39+
end
40+
41+
do_all_shifts_match = all(do_shifts_match_from_tile_index) %#ok<NOPTS>

‎sample-2022-02-01/delete_broken_line_fix_tiles.m

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
sample_date = '2022-02-01' ;
2+
broken_tile_paths = { ...
3+
'2022-02-05/02/02572', ...
4+
'2022-02-05/02/02638', ...
5+
'2022-02-05/02/02646', ...
6+
'2022-02-05/02/02650', ...
7+
'2022-02-05/02/02664', ...
8+
'2022-02-05/02/02668', ...
9+
'2022-02-05/02/02672', ...
10+
'2022-02-05/02/02686', ...
11+
'2022-02-05/02/02690', ...
12+
'2022-02-05/02/02698', ...
13+
'2022-02-05/02/02699', ...
14+
'2022-02-05/02/02704', ...
15+
'2022-02-05/02/02713', ...
16+
'2022-02-05/02/02730', ...
17+
'2022-02-05/02/02734' } ;
18+
19+
pipeline_output_folder_path_template_from_stage_index = { ...
20+
'/nrs/mouselight/pipeline_output/%s/stage_1_line_fix_output', ...
21+
'/nrs/mouselight/pipeline_output/%s/stage_2_classifier_output', ...
22+
'/nrs/mouselight/pipeline_output/%s/stage_3_descriptor_output', ...
23+
'/nrs/mouselight/pipeline_output/%s/stage_4_point_match_output'} ;
24+
pipeline_output_folder_from_stage_index = ...
25+
cellfun(@(template)(sprintf(template, sample_date)), pipeline_output_folder_path_template_from_stage_index, 'UniformOutput', false) ;
26+
27+
stage_count = length(pipeline_output_folder_from_stage_index) ;
28+
for stage_index = 1 : stage_count ,
29+
root_path = pipeline_output_folder_from_stage_index{stage_index}
30+
cellfun(@(tile_path)(system_from_list_with_error_handling({'rm', '-rf', fullfile(root_path, tile_path)})), broken_tile_paths, 'UniformOutput', false) ;
31+
end

‎sample-2022-02-01/exercise_vectorField3D.m

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
load('vectorField3D_test_inputs.mat') ;
2+
do_cold_stitch = false ;
3+
vecfield3D = vectorField3D(params, scopeloc, do_cold_stitch, regpts, scopeparams, curvemodel) ;
4+
% do_cold_stitch = true ;
5+
% vecfield3D = vectorField3D(params, scopeloc, do_cold_stitch, [], [], []) ;

‎sample-2022-02-01/fix_line_shift_in_place_2022_02_01.m

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
sample_date = '2022-02-01'
2+
fluorescence_root_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s/Tiling', sample_date)
3+
sample_memo_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/cluster/Reconstructions/%s', sample_date)
4+
do_use_bqueue = true
5+
do_actually_submit = true
6+
do_run_in_debug_mode = false
7+
8+
fix_line_shift_in_place_for_sample(fluorescence_root_folder_path, sample_memo_folder_path, do_use_bqueue, do_actually_submit, do_run_in_debug_mode) ;

‎sample-2022-02-01/fix_line_shift_in_place_2022_02_01_run_01.out.txt

+5,000
Large diffs are not rendered by default.

‎sample-2022-02-01/mouselight-landmark-matching.out.txt

+74
Large diffs are not rendered by default.

‎sample-2022-02-01/run_all_tiles_through_1234_stages_2022_02_01.m

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
% Set parameters
2+
do_line_fix = true ;
3+
ilastik_project_file_name = '' ;
4+
do_force_computation = false ;
5+
do_use_bsub = true ;
6+
do_actually_submit = true ;
7+
do_run_in_debug_mode = true ;
8+
9+
% Build an index of the paths to raw tiles
10+
sample_date = '2022-02-01' ;
11+
script_folder_path = fileparts(mfilename('fullpath')) ;
12+
raw_root_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s/Tiling', sample_date) ;
13+
sample_memo_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/cluster/Reconstructions/%s', sample_date) ;
14+
line_fix_root_path = sprintf('/nrs/mouselight/pipeline_output/%s/stage_1_line_fix_output', sample_date) ;
15+
p_map_root_path = sprintf('/nrs/mouselight/pipeline_output/%s/stage_2_classifier_output', sample_date) ;
16+
landmark_root_path = sprintf('/nrs/mouselight/pipeline_output/%s/stage_3_descriptor_output', sample_date) ;
17+
z_point_match_root_path = sprintf('/nrs/mouselight/pipeline_output/%s/stage_4_point_match_output', sample_date) ;
18+
19+
20+
% Run the generic script
21+
run_all_tiles_through_1234_stages

‎sample-2022-02-01/run_all_tiles_through_1234_stages_2022_02_01_run_01.out.txt

+77
Large diffs are not rendered by default.

‎sample-2022-02-01/run_all_tiles_through_1234_stages_2022_02_01_run_02.out.txt

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
Using 4 thread(s) on compute node.
2+
{Undefined function or variable 'run_all_tiles_through_1234_stages_2022_02_02'.
3+
}
4+
5+
------------------------------------------------------------
6+
Sender: LSF System <lsfadmin@h07u19>
7+
Subject: Job 118692539: </misc/local/matlab-2019a/bin/matlab -nodisplay -batch "modpath; run_all_tiles_through_1234_stages_2022_02_02"> in cluster <Janelia> Exited
8+
9+
Job </misc/local/matlab-2019a/bin/matlab -nodisplay -batch "modpath; run_all_tiles_through_1234_stages_2022_02_02"> was submitted from host <h07u01> by user <taylora> in cluster <Janelia> at Thu Feb 24 20:48:04 2022
10+
Job was executed on host(s) <4*h07u19>, in queue <local>, as user <taylora> in cluster <Janelia> at Thu Feb 24 20:48:04 2022
11+
</groups/scicompsoft/home/taylora> was used as the home directory.
12+
</groups/mousebrainmicro/mousebrainmicro/scripts/sample-2022-02-01> was used as the working directory.
13+
Started at Thu Feb 24 20:48:04 2022
14+
Terminated at Thu Feb 24 20:48:26 2022
15+
Results reported at Thu Feb 24 20:48:26 2022
16+
17+
Your job looked like:
18+
19+
------------------------------------------------------------
20+
# LSBATCH: User input
21+
/misc/local/matlab-2019a/bin/matlab -nodisplay -batch "modpath; run_all_tiles_through_1234_stages_2022_02_02"
22+
------------------------------------------------------------
23+
24+
Exited with exit code 1.
25+
26+
Resource usage summary:
27+
28+
CPU time : 20.39 sec.
29+
Max Memory : 897 MB
30+
Average Memory : 579.00 MB
31+
Total Requested Memory : 61440.00 MB
32+
Delta Memory : 60543.00 MB
33+
Max Swap : -
34+
Max Processes : 6
35+
Max Threads : 47
36+
Run time : 20 sec.
37+
Turnaround time : 22 sec.
38+
39+
The output (if any) is above this job summary.
40+

‎sample-2022-02-01/run_all_tiles_through_1234_stages_2022_02_01_run_03.out.txt

+84
Large diffs are not rendered by default.

‎sample-2022-02-01/stitch_2022_02_01_alt_no_fc_after_fixing_broken_tiles.m

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
% Specify the input/output folders
2+
sample_date = '2022-02-01'
3+
analysis_tag = 'production-classifier-z-match-count-threshold-50-alt-no-fc-after-fixing-broken-tiles'
4+
do_force_computations = false
5+
do_perform_field_correction = false
6+
do_run_in_debug_mode = false
7+
do_show_visualizations = true
8+
9+
% Define the paths for various things
10+
%script_folder_path = fileparts(mfilename('fullpath'))
11+
12+
raw_tile_root_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s/Tiling', sample_date)
13+
%stage_1_root_folder_path = sprintf('/nrs/mouselight/pipeline_output/%s/stage_1_line_fix_output', sample_date)
14+
%stage_2_root_folder_path = sprintf('/nrs/mouselight/pipeline_output/%s/stage_2_classifier_output', sample_date)
15+
landmark_root_folder_path = sprintf('/nrs/mouselight/pipeline_output/%s/stage_3_descriptor_output', sample_date)
16+
z_match_root_folder_path = sprintf('/nrs/mouselight/pipeline_output/%s/stage_4_point_match_output', sample_date)
17+
18+
sample_memo_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/cluster/Reconstructions/%s', sample_date)
19+
analysis_memo_folder_path = fullfile(sample_memo_folder_path, analysis_tag)
20+
stitching_output_folder_path = fullfile(analysis_memo_folder_path, 'stitching-output')
21+
22+
% Set the options
23+
options = ...
24+
struct('do_force_computations', do_force_computations, ...
25+
'do_perform_field_correction', do_perform_field_correction, ...
26+
'do_run_in_debug_mode', do_run_in_debug_mode, ...
27+
'do_show_visualizations', do_show_visualizations) ;
28+
29+
% Call the function that does the real work
30+
stitch_and_compute_match_errors(raw_tile_root_folder_path, ...
31+
sample_memo_folder_path, ...
32+
analysis_memo_folder_path, ...
33+
landmark_root_folder_path, ...
34+
z_match_root_folder_path, ...
35+
stitching_output_folder_path, ...
36+
options)
37+

‎sample-2022-02-01/stitch_2022_02_01_alt_with_fc_after_fixing_broken_tiles.m

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
% Specify the input/output folders
2+
sample_date = '2022-02-01'
3+
analysis_tag = 'production-classifier-z-match-count-threshold-50-alt-with-fc-after-fixing-broken-tiles'
4+
do_force_computations = false
5+
do_perform_field_correction = true
6+
do_run_in_debug_mode = false
7+
do_show_visualizations = true
8+
9+
% Define the paths for various things
10+
%script_folder_path = fileparts(mfilename('fullpath'))
11+
12+
raw_tile_root_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s/Tiling', sample_date)
13+
%stage_1_root_folder_path = sprintf('/nrs/mouselight/pipeline_output/%s/stage_1_line_fix_output', sample_date)
14+
%stage_2_root_folder_path = sprintf('/nrs/mouselight/pipeline_output/%s/stage_2_classifier_output', sample_date)
15+
landmark_root_folder_path = sprintf('/nrs/mouselight/pipeline_output/%s/stage_3_descriptor_output', sample_date)
16+
z_match_root_folder_path = sprintf('/nrs/mouselight/pipeline_output/%s/stage_4_point_match_output', sample_date)
17+
18+
sample_memo_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/cluster/Reconstructions/%s', sample_date)
19+
analysis_memo_folder_path = fullfile(sample_memo_folder_path, analysis_tag)
20+
stitching_output_folder_path = fullfile(analysis_memo_folder_path, 'stitching-output')
21+
22+
% Set the options
23+
options = ...
24+
struct('do_force_computations', do_force_computations, ...
25+
'do_perform_field_correction', do_perform_field_correction, ...
26+
'do_run_in_debug_mode', do_run_in_debug_mode, ...
27+
'do_show_visualizations', do_show_visualizations) ;
28+
29+
% Call the function that does the real work
30+
stitch_and_compute_match_errors(raw_tile_root_folder_path, ...
31+
sample_memo_folder_path, ...
32+
analysis_memo_folder_path, ...
33+
landmark_root_folder_path, ...
34+
z_match_root_folder_path, ...
35+
stitching_output_folder_path, ...
36+
options)
37+

0 commit comments

Comments
 (0)