Skip to content

Commit ed0fe1f

Browse files
adamltayloradamltaylor
committed
Folded in http://github.com/MouseLightProject/freezing repo as of commit 6f0a07bc, dated 2023-05-24
1 parent 4b19a21 commit ed0fe1f

File tree

59 files changed

+1289
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+1289
-0
lines changed

‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,6 @@ parfor_progress.txt
99
foreground-classifiers/data/
1010
foreground-classifiers/2021-03-17/
1111
RESTORED/
12+
freezing/octree*test*folder
13+
freezing/raw-tiles*test*folder
1214

‎freezing/are_files_same_size_and_md5sum.m

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
function result = are_files_same_size_and_md5sum(file_name_1, file_name_2)
2+
file_size_1 = get_file_size(file_name_1) ;
3+
file_size_2 = get_file_size(file_name_2) ;
4+
if file_size_1 ~= file_size_2 ,
5+
result = false ;
6+
return
7+
end
8+
result = are_md5s_the_same(file_name_1, file_name_2) ;
9+
end

‎freezing/batch_convert_all_mj2_files_to_tif.m

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
function batch_convert_all_mj2_files_to_tif(output_folder_name, ...
2+
input_folder_name, ...
3+
do_submit, ...
4+
do_try, ...
5+
maximum_running_slot_count, ...
6+
submit_host_name)
7+
% Find all .mj2 files in folders with no subfolders, convert them to .tif
8+
9+
slots_per_job = 1 ;
10+
bsub_options = '-P mouselight -W 59 -J thaw' ;
11+
find_and_batch(input_folder_name, ...
12+
@is_a_mj2_and_target_missing, ...
13+
@tif_from_mj2_single_for_find_and_batch, ...
14+
'do_submit', do_submit, ...
15+
'do_try', do_try, ...
16+
'submit_host_name', submit_host_name, ...
17+
'maximum_running_slot_count', maximum_running_slot_count, ...
18+
'slots_per_job', slots_per_job, ...
19+
'bsub_options', bsub_options, ...
20+
'predicate_extra_args', {input_folder_name, output_folder_name}, ...
21+
'batch_function_extra_args', {input_folder_name, output_folder_name}) ;
22+
end

‎freezing/batch_convert_all_tif_files_to_mj2.m

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
function batch_convert_all_tif_files_to_mj2(mj2_folder_name, ...
2+
tif_folder_name, ...
3+
compression_ratio, ...
4+
do_submit, ...
5+
do_try, ...
6+
maximum_running_slot_count, ...
7+
submit_host_name)
8+
% Find all .tif files in folder tree, convert them to .mj2
9+
10+
slots_per_job = 1 ;
11+
bsub_options = '-P mouselight -W 59 -J freeze' ;
12+
find_and_batch(tif_folder_name, ...
13+
@does_file_name_end_in_dot_tif_and_mj2_is_missing, ...
14+
@mj2_from_tif_single_for_find_and_batch, ...
15+
'do_submit', do_submit, ...
16+
'do_try', do_try, ...
17+
'submit_host_name', submit_host_name, ...
18+
'maximum_running_slot_count', maximum_running_slot_count, ...
19+
'slots_per_job', slots_per_job, ...
20+
'bsub_options', bsub_options, ...
21+
'predicate_extra_args', {tif_folder_name, mj2_folder_name}, ...
22+
'batch_function_extra_args', {tif_folder_name, mj2_folder_name, compression_ratio}) ;
23+
end

‎freezing/batch_verify_mj2_files_after_freezing.m

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
function batch_preverify_stack_files_after_freezing(mj2_folder_name, ...
2+
tif_folder_name, ...
3+
do_submit, ...
4+
do_try, ...
5+
submit_host_name, ...
6+
maximum_running_slot_count)
7+
% Makes sure all the big target files are present in the the destination, and
8+
% are similar to their source files.
9+
10+
slots_per_job = 4 ;
11+
bsub_options = '-P mouselight -W 59 -J verify-frozen' ;
12+
find_and_batch(tif_folder_name, ...
13+
@does_need_preverification_after_freezing, ...
14+
@verify_single_mj2_file_after_freezing, ...
15+
'do_submit', do_submit, ...
16+
'do_try', do_try, ...
17+
'submit_host_name', submit_host_name, ...
18+
'maximum_running_slot_count', maximum_running_slot_count, ...
19+
'slots_per_job', slots_per_job, ...
20+
'bsub_options', bsub_options, ...
21+
'predicate_extra_args', {tif_folder_name, mj2_folder_name}, ...
22+
'batch_function_extra_args', {tif_folder_name, mj2_folder_name}) ;
23+
end

‎freezing/batch_verify_tif_files_after_thawing.m

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
function batch_verify_tif_files_after_thawing(output_folder_name, ...
2+
input_folder_name, ...
3+
do_submit, ...
4+
do_try, ...
5+
submit_host_name, ...
6+
maximum_running_slot_count)
7+
% Makes sure all the leaf image files are present in the the destination, and
8+
% are similar to their source files.
9+
10+
slots_per_job = 4 ;
11+
bsub_options = '-P mouselight -W 59 -J verify-thawed-stack' ;
12+
find_and_batch(input_folder_name, ...
13+
@is_mj2_and_lacks_cert_for_output, ...
14+
@verify_single_tif_file_after_thawing, ...
15+
'do_submit', do_submit, ...
16+
'do_try', do_try, ...
17+
'submit_host_name', submit_host_name, ...
18+
'maximum_running_slot_count', maximum_running_slot_count, ...
19+
'slots_per_job', slots_per_job, ...
20+
'bsub_options', bsub_options, ...
21+
'predicate_extra_args', {input_folder_name, output_folder_name}, ...
22+
'batch_function_extra_args', {input_folder_name, output_folder_name}) ;
23+
end

‎freezing/check_certs_after_verifying_freezing.m

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
function check_certs_after_verifying_freezing(tif_root_folder_path, mj2_root_folder_path)
2+
% Makes sure there's a .similar-mj2-exists for each .tif file in
3+
% tif_root_folder_path.
4+
5+
function result = verify_single_tif_file_after_mj2_from_tif_wrapper(tif_root_folder_path, tif_folder_relative_path, tif_file_name, depth) %#ok<INUSD>
6+
result = verify_single_tif_file_after_mj2_from_tif(tif_root_folder_path, tif_folder_relative_path, tif_file_name, mj2_root_folder_path) ;
7+
end
8+
9+
find_and_verify(...
10+
tif_root_folder_path, ...
11+
@does_file_name_end_in_dot_tif, ...
12+
@verify_single_tif_file_after_mj2_from_tif_wrapper, ...
13+
@is_not_the_ktx_folder) ;
14+
end

‎freezing/check_certs_after_verifying_thawing.m

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
function check_certs_after_verifying_thawing(input_root_folder_path, output_root_folder_path)
2+
% Makes sure there's a .similar-tif-exists for each .mj2 file in
3+
% input_root_folder_path.
4+
5+
function result = check_single_cert_after_verifying_thawing_wrapper(root_folder_path, base_folder_relative_path, file_name, depth) %#ok<INUSD>
6+
result = check_single_cert_after_verifying_thawing(root_folder_path, base_folder_relative_path, file_name, output_root_folder_path) ;
7+
end
8+
9+
find_and_verify(...
10+
input_root_folder_path, ...
11+
@does_file_name_end_in_dot_mj2, ...
12+
@check_single_cert_after_verifying_thawing_wrapper, ...
13+
@(varargin)(true)) ;
14+
end

‎freezing/check_single_cert_after_verifying_thawing.m

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
function result = check_single_cert_after_verifying_thawing(...
2+
input_root_folder_path, input_folder_relative_path, input_file_name, output_root_folder_path) %#ok<INUSD>
3+
output_file_name = replace_extension(input_file_name, '.tif') ;
4+
output_file_path = fullfile(output_root_folder_path, input_folder_relative_path, output_file_name) ;
5+
cert_file_path = horzcat(output_file_path, '.is-similar-to-mj2') ;
6+
result = logical( exist(cert_file_path, 'file') ) ;
7+
end

‎freezing/copy_file_for_find_and_batch.m

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
function copy_file_for_find_and_batch(tif_side_file_path, tif_root_folder_name, mj2_root_folder_name, varargin)
2+
relative_file_path_of_tif_side_file = relpath(tif_side_file_path, tif_root_folder_name) ;
3+
relative_file_path_of_mj2_side_file = relative_file_path_of_tif_side_file ;
4+
mj2_side_file_path = fullfile(mj2_root_folder_name, relative_file_path_of_mj2_side_file) ;
5+
mj2_side_folder_path = fileparts2(mj2_side_file_path) ;
6+
ensure_folder_exists(mj2_side_folder_path) ;
7+
copyfile(tif_side_file_path, mj2_side_file_path) ;
8+
end

‎freezing/copy_non_stack_files_during_freezing.m

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
function copy_non_stack_files_during_freezing(output_folder_name, ...
2+
input_folder_name)
3+
% Find all non-.tif files, copy them over to the mj2 side
4+
5+
do_submit = false ;
6+
do_try = true ;
7+
maximum_running_slot_count = 400 ; % doesn't matter
8+
slots_per_job = 1 ; % doesn't matter
9+
bsub_options = '-P mouselight -W 59 -J copy' ; % doesn't matter
10+
submit_host_name = '' ;
11+
12+
function result = predicate_function(tif_side_file_path)
13+
result = is_a_non_tif_and_target_missing(tif_side_file_path, input_folder_name, output_folder_name) ;
14+
end
15+
16+
function batch_function(tif_side_file_path)
17+
copy_file_for_find_and_batch(tif_side_file_path, input_folder_name, output_folder_name) ;
18+
end
19+
20+
find_and_batch(input_folder_name, ...
21+
@predicate_function, ...
22+
@batch_function, ...
23+
'do_submit', do_submit, ...
24+
'do_try', do_try, ...
25+
'submit_host_name', submit_host_name, ...
26+
'maximum_running_slot_count', maximum_running_slot_count, ...
27+
'slots_per_job', slots_per_job, ...
28+
'bsub_options', bsub_options, ...
29+
'folder_predicate_function', @is_not_the_ktx_folder) ;
30+
end

‎freezing/copy_non_stack_files_during_thawing.m

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
function copy_non_stack_files_during_thawing(output_folder_name, ...
2+
input_folder_name)
3+
% Find all non-.mj2 files, copy them over to the tif side
4+
5+
do_submit = false ;
6+
do_try = true ;
7+
maximum_running_slot_count = 400 ; % doesn't matter
8+
slots_per_job = 1 ; % doesn't matter
9+
bsub_options = '-P mouselight -W 59 -J copy' ; % doesn't matter
10+
submit_host_name = '' ;
11+
12+
function result = predicate_function(mj2_side_file_path)
13+
result = is_a_non_mj2_and_target_missing(mj2_side_file_path, input_folder_name, output_folder_name) ;
14+
end
15+
16+
function batch_function(mj2_side_file_path)
17+
copy_file_for_find_and_batch(mj2_side_file_path, input_folder_name, output_folder_name) ;
18+
end
19+
20+
find_and_batch(input_folder_name, ...
21+
@predicate_function, ...
22+
@batch_function, ...
23+
'do_submit', do_submit, ...
24+
'do_try', do_try, ...
25+
'submit_host_name', submit_host_name, ...
26+
'maximum_running_slot_count', maximum_running_slot_count, ...
27+
'slots_per_job', slots_per_job, ...
28+
'bsub_options', bsub_options) ;
29+
end

‎freezing/create_octree_test_folder.m

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
output_folder_name = '/groups/mousebrainmicro/mousebrainmicro/scripts/octree-freezing/octree-rebuilding-test-input-folder' ;
2+
input_folder_name = '/nrs/mouselight/SAMPLES/2023-02-23' ;
3+
max_depth = 2 ;
4+
do_submit = true ;
5+
do_try = false ;
6+
maximum_running_slot_count = 400 ;
7+
slots_per_job = 1 ;
8+
bsub_options = '-P mouselight -W 59 -J copy' ;
9+
submit_host_name = 'login2.int.janelia.org' ;
10+
11+
% Delete the test output folder
12+
reset_for_test(output_folder_name) ;
13+
14+
% Do the copy
15+
find_and_batch(input_folder_name, ...
16+
@(file_path)(true), ...
17+
@copy_file_for_find_and_batch, ...
18+
'do_submit', do_submit, ...
19+
'do_try', do_try, ...
20+
'submit_host_name', submit_host_name, ...
21+
'maximum_running_slot_count', maximum_running_slot_count, ...
22+
'slots_per_job', slots_per_job, ...
23+
'bsub_options', bsub_options, ...
24+
'max_depth', max_depth, ...
25+
'predicate_extra_args', cell(1,0), ...
26+
'batch_function_extra_args', {input_folder_name, output_folder_name}) ;

‎freezing/does_folder_have_subfolders.m

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
function result = does_folder_have_subfolders(folder_name)
2+
[name_from_thing_index, is_folder_from_thing_index] = simple_dir(folder_name) ; %#ok<ASGLU>
3+
result = any(is_folder_from_thing_index) ;
4+
end

‎freezing/does_is_similar_to_mj2_check_file_exist.m

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
function result = does_is_similar_to_mj2_check_file_exist(tif_file_path, mj2_root_path, tif_root_path)
2+
mj2_relative_path = relpath(tif_file_path, mj2_root_path) ;
3+
tif_relative_path = replace_extension(mj2_relative_path, '.tif') ;
4+
tif_file_path = fullfile(tif_root_path, tif_relative_path) ;
5+
check_file_path = horzcat(tif_file_path, '.is-similar-to-mj2') ;
6+
result = logical(exist(check_file_path, 'file')) ;
7+
end

‎freezing/does_is_similar_to_tif_check_file_exist.m

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
function result = does_is_similar_to_tif_check_file_exist(tif_file_path, tif_root_path, mj2_root_path)
2+
tif_relative_path = relpath(tif_file_path, tif_root_path) ;
3+
mj2_relative_path = replace_extension(tif_relative_path, '.mj2') ;
4+
mj2_file_path = fullfile(mj2_root_path, mj2_relative_path) ;
5+
check_file_path = horzcat(mj2_file_path, '.is-similar-to-tif') ;
6+
result = logical(exist(check_file_path, 'file')) ;
7+
end

‎freezing/does_need_preverification_after_freezing.m

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
function result = does_need_preverification_after_freezing(tif_file_path, tif_root_path, mj2_root_path)
2+
result = does_file_name_end_in_dot_tif(tif_file_path) && ...
3+
~does_is_similar_to_tif_check_file_exist(tif_file_path, tif_root_path, mj2_root_path) ;
4+
end

‎freezing/find_and_count.m

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
function result = find_and_count(root_folder_path, is_file_a_keeper_predicate, folder_filter_function)
2+
seed = 0 ; % for counting, start with a count of zero
3+
4+
function result = filter(root_folder_path, base_folder_relative_path, file_name, depth, is_this_file_a_folder) %#ok<INUSD>
5+
result = feval(is_file_a_keeper_predicate, root_folder_path, base_folder_relative_path, file_name) ;
6+
end
7+
8+
result = file_accumulate_map_filter(root_folder_path, seed, @accumulator, @mapper, @filter, folder_filter_function) ;
9+
end
10+
11+
function result = accumulator(result_so_far, file_value)
12+
% For counting, just add the file_value to the ongoing count
13+
result = result_so_far + file_value ;
14+
end
15+
16+
function result = mapper(root_folder_path, base_folder_relative_path, file_name, depth) %#ok<INUSD>
17+
% For counting, just want to return one for each file that gets past the
18+
% filter
19+
result = 1 ;
20+
end
21+
22+
23+
% function result = find_and_count(base_folder_path, is_file_a_keeper_predicate, varargin)
24+
% [folder_predicate_function] = ...
25+
% parse_keyword_args(...
26+
% varargin, ...
27+
% 'folder_predicate_function', @(folder_path, depth)(true)) ;
28+
%
29+
% result = ...
30+
% find_and_count_helper(base_folder_path, ...
31+
% is_file_a_keeper_predicate, ...
32+
% folder_predicate_function, ...
33+
% 0, ...
34+
% 0) ;
35+
% end
36+
%
37+
%
38+
%
39+
% function result = ...
40+
% find_and_count_helper(base_folder_path, ...
41+
% is_file_a_keeper_predicate, ...
42+
% folder_predicate_function, ...
43+
% depth, ...
44+
% incoming_count)
45+
% count_so_far = incoming_count ;
46+
% [file_names, is_file_a_folder] = simple_dir(base_folder_path) ;
47+
% file_count = length(file_names) ;
48+
% for i = 1 : file_count ,
49+
% file_name = file_names{i} ;
50+
% is_this_file_a_folder = is_file_a_folder(i) ;
51+
% file_path = fullfile(base_folder_path, file_name) ;
52+
% if is_this_file_a_folder ,
53+
% % if a folder satisfying predicate function, recurse
54+
% if feval(folder_predicate_function, file_path, depth) ,
55+
% count_so_far = ...
56+
% find_and_count_helper(...
57+
% file_path, ...
58+
% is_file_a_keeper_predicate, ...
59+
% folder_predicate_function, ...
60+
% depth+1, ...
61+
% count_so_far) ;
62+
% end
63+
% else
64+
% if feval(is_file_a_keeper_predicate, file_path) ,
65+
% count_so_far = count_so_far + 1 ;
66+
% end
67+
% end
68+
% end
69+
% result = count_so_far ;
70+
% end

‎freezing/fraction_variance_explained.m

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
function result = fraction_variance_explained(output_stack, input_stack)
2+
% Compute the fraction of the variance in the input stack which is explained
3+
% by the output stack. This is just a gross check that the output is close to
4+
% the input.
5+
if isequal(class(output_stack), class(input_stack)) && isequal(size(output_stack), size(input_stack)) ,
6+
if all(all(all(output_stack==input_stack))) ,
7+
result = 1 ;
8+
else
9+
output_stack_serial = double(output_stack(:)) ;
10+
input_stack_serial = double(input_stack(:)) ;
11+
input_mean = mean(input_stack_serial) ;
12+
13+
residual_variance = mean( (output_stack_serial-input_stack_serial).^2 ) ;
14+
%sqrt_residual_variance = sqrt(residual_variance)
15+
total_variance = mean( (input_stack_serial-input_mean).^2 ) ;
16+
%sqrt_total_variance = sqrt(total_variance)
17+
18+
if residual_variance == 0 ,
19+
% If residual variance is 0, want the output to be 1, even if total variance
20+
% is also zero. (The general expression below will be nan if both are zero.)
21+
result = 1 ;
22+
else
23+
result = 1 - residual_variance/total_variance ;
24+
% this can actually be negative, since the output_stack is not a generally a regression
25+
end
26+
end
27+
else
28+
result = 0 ;
29+
end
30+
end

‎freezing/freeze_2022_05_30.m

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
sample_date = '2022-05-30' ;
2+
mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/%s', sample_date) ;
3+
tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ;
4+
compression_ratio = 10 ;
5+
do_run_on_cluster = true ;
6+
do_try = true ;
7+
maximum_running_slot_count = 500 ;
8+
submit_host_name = if_not_a_submit_host('login2.int.janelia.org') ;
9+
10+
freeze_mouselight_folder(...
11+
mj2_output_folder_path, ...
12+
tif_input_folder_path, ...
13+
compression_ratio, ...
14+
do_run_on_cluster, ...
15+
do_try, ...
16+
maximum_running_slot_count, ...
17+
submit_host_name) ;

0 commit comments

Comments
 (0)