Merge pull request #69 from DiamondLightSource/memfix

Memory fixing and reorder of pipeline dirrectives for 360deg scans

Author: dkazanc

httomo_backends/methods_database/packages/backends/httomolibgpu/httomolibgpu.yaml

@@ -7,7 +7,7 @@ misc:
       save_result_default: False
       padding: True
       memory_gpu:
-        multiplier: 2.1
+        multiplier: 2.5
         method: direct
     remove_outlier:
       pattern: all
@@ -16,7 +16,7 @@ misc:
       save_result_default: False
       padding: True
       memory_gpu:
-        multiplier: 2.1
+        multiplier: 2.5
         method: direct
   denoise:
     total_variation_ROF:
@@ -198,7 +198,7 @@ recon:
       save_result_default: False
       padding: False
       memory_gpu:
-        multiplier: 0
+        multiplier: 2
         method: direct
     find_center_pc:
       pattern: projection

httomo_backends/pipelines_full/deg360_distortion_FBP3d_tomobar_directive.yaml

@@ -1,11 +1,11 @@
 - method: standard_tomo
   module_path: httomo.data.hdf.loaders
+- method: find_center_360
+  module_path: httomolibgpu.recon.rotation
 - method: normalize
   module_path: httomolibgpu.prep.normalize
 - method: distortion_correction_proj_discorpy
   module_path: httomolibgpu.prep.alignment
-- method: find_center_360
-  module_path: httomolibgpu.recon.rotation
 - method: sino_360_to_180
   module_path: httomolibgpu.misc.morph
 - method: remove_stripe_based_sorting

httomo_backends/pipelines_full/deg360_paganin_FBP3d_tomobar_directive.yaml

@@ -1,9 +1,9 @@
 - method: standard_tomo
   module_path: httomo.data.hdf.loaders
-- method: normalize
-  module_path: httomolibgpu.prep.normalize  
 - method: find_center_360
   module_path: httomolibgpu.recon.rotation
+- method: normalize
+  module_path: httomolibgpu.prep.normalize  
 - method: sino_360_to_180
   module_path: httomolibgpu.misc.morph
 - method: remove_stripe_based_sorting

httomo_backends/scripts/yaml_pipelines_generator.py

@@ -33,12 +33,14 @@
 
 CS = ruamel.yaml.comments.CommentedSeq  # defaults to block style
 
+
 class SweepRange:
     """SweepRange class."""
 
     def __init__(self, start, stop, step):
         self._start, self._stop, self._step = start, stop, step
 
+
 def __sweeprange_representer(
     dumper: yaml.SafeDumper, swp: SweepRange
 ) -> yaml.nodes.MappingNode:
@@ -52,6 +54,7 @@ def __sweeprange_representer(
         },
     )
 
+
 class SweepManual:
     """SweepManual class."""
 

tests/test_httomolibgpu.py

@@ -113,13 +113,11 @@ def test_normalize_memoryhook(flats, darks, ensure_clean_memory, dtype, slices):
 
 
 @pytest.mark.parametrize("dtype", ["uint16"])
-@pytest.mark.parametrize("slices", [151, 321])
+@pytest.mark.parametrize("slices", [10, 20])
 @pytest.mark.cupy
 def test_remove_outlier_memoryhook(flats, ensure_clean_memory, dtype, slices):
     hook = MaxMemoryHook()
-    data = cp.random.random_sample(
-        (slices, flats.shape[1], flats.shape[2]), dtype=np.float32
-    )
+    data = cp.random.random_sample((slices, 2560, 2560), dtype=np.float32)
     if dtype == "uint16":
         data = data.astype(np.uint16)
     with hook:
@@ -148,7 +146,7 @@ def test_remove_outlier_memoryhook(flats, ensure_clean_memory, dtype, slices):
     # the estimated_memory_mb should be LARGER or EQUAL to max_mem_mb
     # the resulting percent value should not deviate from max_mem on more than 20%
     assert estimated_memory_mb >= max_mem_mb
-    assert percents_relative_maxmem <= 20
+    assert percents_relative_maxmem <= 25
 
 
 @pytest.mark.parametrize("slices", [3, 10, 20])