Skip to content

Commit 3945e2f

Browse files
adamltayloradamltaylor
committed
Added check_tiles.m and dependencies
1 parent 9983437 commit 3945e2f

6 files changed

+331
-0
lines changed

‎check_tiles/check_classifier_tile.m

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
function [problematic_stage_output_tile_relative_paths, problematic_stage_output_tile_messages] = ...
2+
check_classifier_tile(stage_output_tile_folder_path, ...
3+
putative_stage_output_tile_relative_path, ...
4+
nominal_raw_tif_file_size, ...
5+
nominal_tile_shape_yxz, ...
6+
problematic_stage_output_tile_relative_paths, ...
7+
problematic_stage_output_tile_messages) %#ok<INUSL>
8+
9+
putative_stage_output_tile_absolute_path = fullfile(stage_output_tile_folder_path, ...
10+
putative_stage_output_tile_relative_path) ;
11+
nominal_tile_shape_xyz = nominal_tile_shape_yxz([2 1 3]) ;
12+
try
13+
info = h5info(putative_stage_output_tile_absolute_path, '/') ;
14+
did_get_info = true ;
15+
catch err
16+
if isequal(err.identifier, 'MATLAB:imagesci:h5info:fileOpenErr') ,
17+
problematic_stage_output_tile_relative_paths = ...
18+
horzcat(problematic_stage_output_tile_relative_paths, putative_stage_output_tile_relative_path) ;
19+
problematic_stage_output_tile_messages = ...
20+
horzcat(problematic_stage_output_tile_messages, ...
21+
sprintf('Trying to read %s caused the error %s', ...
22+
putative_stage_output_tile_relative_path, err.message)) ;
23+
did_get_info = false ;
24+
else
25+
rethrow(err) ;
26+
end
27+
end
28+
if did_get_info ,
29+
thing_names = {info.Datasets.Name} ;
30+
% should be only one
31+
if isempty(thing_names) ,
32+
problematic_stage_output_tile_relative_paths = ...
33+
horzcat(problematic_stage_output_tile_relative_paths, ...
34+
putative_stage_output_tile_relative_path) ;
35+
problematic_stage_output_tile_messages = ...
36+
horzcat(problematic_stage_output_tile_messages, ...
37+
sprintf('%s contains zero datasets', putative_stage_output_tile_relative_path)) ;
38+
else
39+
thing_name = thing_names{1} ;
40+
%dataset = h5read(putative_stage_output_tile_absolute_path, sprintf('/%s', thing_name)) ;
41+
dataset_info = h5info(putative_stage_output_tile_absolute_path, sprintf('/%s', thing_name)) ;
42+
raw_shape_xyz = dataset_info.Dataspace.Size ;
43+
% Sometimes the shape has a leading singleton dimension, so trim that off
44+
if length(raw_shape_xyz) == 4 && raw_shape_xyz(1)==1 ,
45+
shape_xyz = raw_shape_xyz(end-2:end) ;
46+
else
47+
shape_xyz = raw_shape_xyz ;
48+
end
49+
dtype = dataset_info.Datatype.Type ;
50+
if ~isequal(shape_xyz, nominal_tile_shape_xyz) ,
51+
problematic_stage_output_tile_relative_paths = ...
52+
horzcat(problematic_stage_output_tile_relative_paths, ...
53+
putative_stage_output_tile_relative_path) ;
54+
problematic_stage_output_tile_messages = ...
55+
horzcat(problematic_stage_output_tile_messages, ...
56+
sprintf('%s is the wrong shape: shape (xyz) is %s', ...
57+
putative_stage_output_tile_relative_path, ...
58+
num2str(shape_xyz))) ;
59+
elseif ~isequal(dtype, 'H5T_STD_U16LE') ,
60+
problematic_stage_output_tile_relative_paths = ...
61+
horzcat(problematic_stage_output_tile_relative_paths, ...
62+
putative_stage_output_tile_relative_path) ;
63+
problematic_stage_output_tile_messages = ...
64+
horzcat(problematic_stage_output_tile_messages, ...
65+
sprintf('%s is the wrong data type, type is %s', putative_stage_output_tile_relative_path, dtype)) ;
66+
end
67+
end
68+
end
69+
end

‎check_tiles/check_raw_tile.m

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
function [problematic_stage_output_tile_relative_paths, problematic_stage_output_tile_messages] = ...
2+
check_raw_tile(stage_root_folder_path, ...
3+
tile_file_relative_path, ...
4+
nominal_raw_tif_file_size, ...
5+
nominal_tile_shape_yxz, ...
6+
problematic_stage_output_tile_relative_paths, ...
7+
problematic_stage_output_tile_messages)
8+
9+
tile_file_absolute_path = fullfile(stage_root_folder_path, tile_file_relative_path) ;
10+
tile_file_size = get_file_size(tile_file_absolute_path) ;
11+
if isequal(tile_file_size, nominal_raw_tif_file_size) ,
12+
% all is well, do nothing
13+
else
14+
% If we get here, it might be a 'short stack', one with too few z slices, or
15+
% it might be a patched short stack (which is fine, but has
16+
% different metadata than untouched stacks), so we investigate further...
17+
try
18+
[tile_shape, bits_per_pixel] = get_tiff_shape(tile_file_absolute_path) ;
19+
catch me %#ok<NASGU>
20+
problematic_stage_output_tile_relative_paths = ...
21+
horzcat(problematic_stage_output_tile_relative_paths, ...
22+
tile_file_relative_path) ;
23+
problematic_stage_output_tile_messages = ...
24+
horzcat(problematic_stage_output_tile_messages, ...
25+
sprintf('Could not obtain shape, bits-per-pixel of %s', ...
26+
tile_file_relative_path)) ;
27+
return
28+
end
29+
if ~isequal(tile_shape, nominal_tile_shape_yxz) ,
30+
problematic_stage_output_tile_relative_paths = ...
31+
horzcat(problematic_stage_output_tile_relative_paths, ...
32+
tile_file_relative_path) ;
33+
problematic_stage_output_tile_messages = ...
34+
horzcat(problematic_stage_output_tile_messages, ...
35+
sprintf('%s is the wrong shape, shape (yxz) is [%d %d %d]', ...
36+
tile_file_relative_path, ...
37+
tile_shape)) ;
38+
else
39+
if bits_per_pixel ~= 16 ,
40+
problematic_stage_output_tile_relative_paths = ...
41+
horzcat(problematic_stage_output_tile_relative_paths, ...
42+
tile_file_relative_path) ;
43+
problematic_stage_output_tile_messages = ...
44+
horzcat(problematic_stage_output_tile_messages, ...
45+
sprintf('%s has %d bits per pixel, should have 16', ...
46+
tile_file_relative_path, ...
47+
bits_per_pixel)) ;
48+
end
49+
end
50+
end
51+
end

‎check_tiles/check_tiles.m

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
function check_tiles(sample_date_as_string, do_show_progress)
2+
if ~exist('do_show_progress', 'var') || isempty(do_show_progress) ,
3+
do_show_progress = false ;
4+
end
5+
6+
raw_tile_folder_path_template = '/groups/mousebrainmicro/mousebrainmicro/data/%s/Tiling' ;
7+
raw_tile_folder_path = sprintf(raw_tile_folder_path_template, sample_date_as_string) ;
8+
% Sometimes they use a "Tiling" folder, sometimes not...
9+
if ~exist(raw_tile_folder_path, 'file') ,
10+
raw_tile_folder_path_template = '/groups/mousebrainmicro/mousebrainmicro/data/acquisition/%s' ;
11+
raw_tile_folder_path = sprintf(raw_tile_folder_path_template, sample_date_as_string) ;
12+
end
13+
pipeline_output_folder_path_template_from_stage_index = {raw_tile_folder_path_template, ...
14+
'/nrs/mouselight/pipeline_output/%s/stage_1_line_fix_output', ...
15+
'/nrs/mouselight/pipeline_output/%s/stage_2_classifier_output', ...
16+
'/nrs/mouselight/pipeline_output/%s/stage_3_descriptor_output', ...
17+
'/nrs/mouselight/pipeline_output/%s/stage_4_point_match_output'} ;
18+
stage_name_from_index = {'raw', 'line-fix', 'classifier', 'descriptor', 'point-match'} ;
19+
20+
% Get a list of all the raw tiles for channel 0.
21+
% Each tile is represented by its relative path within raw_tile_folder_path
22+
fprintf('Scanning filesystem for raw tile folders...\n') ;
23+
raw_tile_folder_relative_paths = collect_raw_tile_folder_relative_paths(raw_tile_folder_path) ;
24+
fprintf('Done scanning filesystem for raw tile folders.\n') ;
25+
26+
% Report the number of tiles
27+
raw_tile_count = length(raw_tile_folder_relative_paths) ;
28+
fprintf('Raw tile folder count: %d\n', raw_tile_count) ;
29+
if raw_tile_count==0 ,
30+
error('There are no raw tiles!') ;
31+
end
32+
33+
% Get the tile shape
34+
first_raw_tile_folder_relative_path = raw_tile_folder_relative_paths{1} ;
35+
[~,first_raw_tile_folder_leaf_name] = fileparts2(first_raw_tile_folder_relative_path) ;
36+
first_raw_tile_channel_0_file_relative_path = fullfile(first_raw_tile_folder_relative_path, sprintf('%s-ngc.0.tif', first_raw_tile_folder_leaf_name)) ;
37+
first_raw_tile_channel_0_file_absolute_path = fullfile(raw_tile_folder_path, first_raw_tile_channel_0_file_relative_path) ;
38+
stack = read_16bit_grayscale_tif(first_raw_tile_channel_0_file_absolute_path) ;
39+
nominal_tile_shape_yxz = size(stack) ;
40+
nominal_raw_tif_file_size = get_file_size(first_raw_tile_channel_0_file_absolute_path) ;
41+
fprintf('Nominal tile shape (yxz) is: [%d %d %d]\n', nominal_tile_shape_yxz) ;
42+
fprintf('Nominal uncompressed tiff file size is: %d\n', nominal_raw_tif_file_size) ;
43+
44+
stage_count = length(stage_name_from_index) ;
45+
for stage_index = 1 : stage_count ,
46+
stage_name = stage_name_from_index{stage_index} ;
47+
pipeline_output_folder_path_template = pipeline_output_folder_path_template_from_stage_index{stage_index} ;
48+
stage_pipeline_output_tile_folder_path = sprintf(pipeline_output_folder_path_template, sample_date_as_string) ;
49+
problematic_stage_output_file_relative_paths = cell(1,0) ;
50+
problematic_stage_output_file_messages = cell(1,0) ;
51+
fprintf('\n')
52+
fprintf('Checking stage %s...\n', stage_name) ;
53+
if do_show_progress ,
54+
progress_bar = progress_bar_object(raw_tile_count) ;
55+
end
56+
for raw_tile_index = 1 : raw_tile_count ,
57+
raw_tile_folder_relative_path = raw_tile_folder_relative_paths{raw_tile_index} ;
58+
[~,leaf_folder_name] = fileparts2(raw_tile_folder_relative_path) ;
59+
if isequal(stage_name, 'classifier') ,
60+
putative_stage_output_tile_file_name_template = sprintf('%s-prob.%%d.h5', leaf_folder_name) ;
61+
are_channels = true ;
62+
elseif isequal(stage_name, 'descriptor') ,
63+
putative_stage_output_tile_file_name_template = sprintf('%s-desc.%%d.txt', leaf_folder_name) ;
64+
are_channels = true ;
65+
elseif isequal(stage_name, 'point-match') ,
66+
%putative_stage_output_tile_file_name_template = 'match-Z.mat' ;
67+
putative_stage_output_tile_file_name_template = 'channel-%d-match-Z.mat' ;
68+
are_channels = true ;
69+
else
70+
putative_stage_output_tile_file_name_template = sprintf('%s-ngc.%%d.tif', leaf_folder_name) ;
71+
are_channels = true ;
72+
end
73+
if are_channels ,
74+
for channel_index = 0:1 ,
75+
putative_stage_output_tile_file_name = sprintf(putative_stage_output_tile_file_name_template, channel_index) ;
76+
putative_stage_output_tile_relative_path = fullfile(raw_tile_folder_relative_path, ...
77+
putative_stage_output_tile_file_name) ;
78+
putative_stage_output_tile_absolute_path = fullfile(stage_pipeline_output_tile_folder_path, ...
79+
putative_stage_output_tile_relative_path) ;
80+
if exist(putative_stage_output_tile_absolute_path, 'file') ,
81+
if isequal(stage_name, 'raw') ,
82+
[problematic_stage_output_file_relative_paths, problematic_stage_output_file_messages] = ...
83+
check_raw_tile(stage_pipeline_output_tile_folder_path, ...
84+
putative_stage_output_tile_relative_path, ...
85+
nominal_raw_tif_file_size, ...
86+
nominal_tile_shape_yxz, ...
87+
problematic_stage_output_file_relative_paths, ...
88+
problematic_stage_output_file_messages) ;
89+
elseif isequal(stage_name, 'classifier') ,
90+
[problematic_stage_output_file_relative_paths, problematic_stage_output_file_messages] = ...
91+
check_classifier_tile(stage_pipeline_output_tile_folder_path, ...
92+
putative_stage_output_tile_relative_path, ...
93+
nominal_raw_tif_file_size, ...
94+
nominal_tile_shape_yxz, ...
95+
problematic_stage_output_file_relative_paths, ...
96+
problematic_stage_output_file_messages) ;
97+
end
98+
else
99+
problematic_stage_output_file_relative_paths = ...
100+
horzcat(problematic_stage_output_file_relative_paths, ...
101+
putative_stage_output_tile_relative_path) ; %#ok<AGROW>
102+
problematic_stage_output_file_messages = ...
103+
horzcat(problematic_stage_output_file_messages, ...
104+
sprintf('%s is missing', putative_stage_output_tile_relative_path)) ; %#ok<AGROW>
105+
end
106+
end
107+
else
108+
% If no channels for this stage, the name is jus the
109+
% template
110+
putative_stage_output_tile_file_name = putative_stage_output_tile_file_name_template ;
111+
putative_stage_output_tile_relative_path = fullfile(raw_tile_folder_relative_path, ...
112+
putative_stage_output_tile_file_name) ;
113+
putative_stage_output_tile_absolute_path = fullfile(stage_pipeline_output_tile_folder_path, ...
114+
putative_stage_output_tile_relative_path) ;
115+
if ~exist(putative_stage_output_tile_absolute_path, 'file') ,
116+
problematic_stage_output_file_relative_paths = ...
117+
horzcat(problematic_stage_output_file_relative_paths, ...
118+
putative_stage_output_tile_relative_path) ; %#ok<AGROW>
119+
problematic_stage_output_file_messages = ...
120+
horzcat(problematic_stage_output_file_messages, ...
121+
sprintf('%s is missing', putative_stage_output_tile_relative_path)) ; %#ok<AGROW>
122+
end
123+
end
124+
125+
% Update the progress bar
126+
if do_show_progress ,
127+
progress_bar.update(raw_tile_index)
128+
end
129+
end
130+
131+
problematic_file_count = length(problematic_stage_output_file_relative_paths) ;
132+
fprintf('\n') ;
133+
fprintf('Problematic %s file count: %d\n', stage_name, problematic_file_count) ;
134+
problematic_files_to_show_count = min(problematic_file_count,inf) ;
135+
for index = 1 : problematic_files_to_show_count ,
136+
problematic_stage_output_file_relative_path = problematic_stage_output_file_relative_paths{index} ;
137+
problematic_stage_output_file_message = problematic_stage_output_file_messages{index} ;
138+
% if stage_name == 'raw':
139+
% problematic_stage_output_tile_absolute_path = os.path.join(raw_tile_folder_path,
140+
% problematic_stage_output_tile_relative_path)
141+
% shape = get_tiff_shape(problematic_stage_output_tile_absolute_path)
142+
% print(' problematic: %s, shape (zyx) is %s' % (problematic_stage_output_tile_relative_path, shape) )
143+
% else:
144+
fprintf(' problematic: %s: %s\n', problematic_stage_output_file_relative_path, problematic_stage_output_file_message) ;
145+
end
146+
if problematic_files_to_show_count < problematic_file_count ,
147+
problematic_files_now_shown_count = problematic_file_count - problematic_files_to_show_count ;
148+
fprintf(' problematic: (%d more files)\n', problematic_files_now_shown_count) ;
149+
end
150+
end
151+
end
152+
153+

‎check_tiles/collect_raw_tile_folder_relative_paths.m

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
function result = collect_raw_tile_folder_relative_paths(raw_tile_folder_path)
2+
entries = simple_dir(raw_tile_folder_path) ;
3+
is_date = ~cellfun(@isempty, regexp(entries, '^\d\d\d\d-\d\d-\d\d$')) ;
4+
date_dir_entries = entries(is_date) ;
5+
raw_tile_file_relative_paths = cell(1,0) ;
6+
for i = 1 : length(date_dir_entries) ,
7+
date_dir_entry = date_dir_entries{i} ;
8+
raw_tile_file_relative_paths_for_date = ...
9+
collect_raw_tile_folder_relative_paths_from_date_subfolder(raw_tile_folder_path, date_dir_entry) ;
10+
raw_tile_file_relative_paths = horzcat(raw_tile_file_relative_paths, raw_tile_file_relative_paths_for_date) ; %#ok<AGROW>
11+
end
12+
result = natsort(raw_tile_file_relative_paths) ;
13+
end

0 commit comments

Comments
 (0)