JP-3243: Updates to jump detection for snowball/shower enhancements (#…

…174) * changes for charge spilling from saturated core * Update jump.py * Update jump.py * Update jump.py * Update saturation.py * Update saturation.py * use ellipses for all jumps * Update jump.py * Update jump.py * Update jump.py * Update jump.py * Update jump.py * Update jump.py * Update jump.py * Update jump.py * Update jump.py * Update jump.py * fix sat expand * fix inside ellipse test * Update jump.py * Update jump.py * Update jump.py * tests etc * return new dgq for sat expend * fix super expanding ellipses * Update jump.py * Update jump.py * wrong shape for edge too many expands due to edge being wrong * update edge parameter * remore print * Update saturation.py * fix miri showers * updates * Update twopoint_difference.py * Update twopoint_difference.py * retro fit sigman_clip * Update twopoint_difference.py * Update twopoint_difference.py * updates for test * Update twopoint_difference.py * fix multint * tests * Update twopoint_difference.py * fixed axis=1 to axis=0,1 * Update twopoint_difference.py * Update twopoint_difference.py * Update twopoint_difference.py print statements * fix flagging fix flagging * Update jump.py * Update twopoint_difference.py * type problem * Update twopoint_difference.py * Update twopoint_difference.py * Update twopoint_difference.py * Update twopoint_difference.py * Update twopoint_difference.py * Update twopoint_difference.py * Update twopoint_difference.py * Update twopoint_difference.py * Update twopoint_difference.py * Update twopoint_difference.py * Update twopoint_difference.py * Update twopoint_difference.py * Update twopoint_difference.py * Update twopoint_difference.py * Update twopoint_difference.py * Update twopoint_difference.py * test sigma clip * Update twopoint_difference.py * Update jump.py * Update twopoint_difference.py * Update twopoint_difference.py * add total CR count * Update twopoint_difference.py * Update jump.py * Update jump.py * Update twopoint_difference.py * Update jump.py * Update jump.py * Update jump.py * Update jump.py * Update jump.py * Update twopoint_difference.py * Update jump.py * Update jump.py * Update jump.py * Update twopoint_difference.py * Update twopoint_difference.py * tests * Update jump.py * lower sig clip thres * Update twopoint_difference.py * Update twopoint_difference.py * Update jump.py * Update twopoint_difference.py * Update twopoint_difference.py * group diffs * Update jump.py * Update jump.py * Update jump.py * Update test_twopoint_difference.py * Update twopoint_difference.py * updates * updates * new test * Update twopoint_difference.py * Update jump.py * Update jump.py * change to nints for min groups * Update test_twopoint_difference.py * fixes * Update twopoint_difference.py * Update twopoint_difference.py * update * Update twopoint_difference.py * update * Update twopoint_difference.py * Update twopoint_difference.py * update * Update twopoint_difference.py * cleanup * Update jump.py * remove fits * add stddev output * updates * test updates * Reduce log output * Update twopoint_difference.py * Update twopoint_difference.py * Update twopoint_difference.py * Update twopoint_difference.py * Update twopoint_difference.py * Update jump.py * Update test_twopoint_difference.py * Add sigma clip to shower detection * remove or mark as skip extraneous tests * remove ds_store * remove DS_Store * Delete .DS_Store * Delete .DS_Store * Update CHANGES.rst * unpacked the use of shape to be named variables * replacing shape with meaningful variable names * fix style issues * Update test_twopoint_difference.py * Delete setup.cfg * Fix missing loop in flag 4 neighbors I added a loop over integrations that was missing due to the post CR flagging being done once for all ints. * Update twopoint_difference.py Remove loop over integrations and use where to get integration. * Update CHANGES.rst * Update CHANGES.rst * Update src/stcal/jump/jump.py * Update src/stcal/jump/jump.py * Update twopoint_difference.py Resolve comments on PR. --------- Co-authored-by: Howard Bushouse <[email protected]>
spacetelescope · Jun 27, 2023 · 644392a · 644392a
1 parent f44e2f4
commit 644392a
Show file tree

Hide file tree

Showing 9 changed files with 550 additions and 212 deletions.
diff --git a/.gitignore b/.gitignore
@@ -143,4 +143,4 @@ src/stcal/_version.py
 
 # auto-generated API docs
 docs/source/api
-.DS_Store
+.DS_Store
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -4,13 +4,28 @@
 Bug Fixes
 ---------
 
+jump
+~~~~
+
+- Updated the jump detection to switch to using the numpy sigmaclip routine to
+  find the actual rms across integrations when there are at least 101 integrations
+  in the exposure. This still allows cosmic rays and snowballs/showers to be flagged
+  without being affected by slope variations due to either brigher-fatter/charge-spilling
+  or errors in the nonlinearity correction.
+  Also added the counting of the number of cosmic rays and snowballs/showers that
+  is then placed in the FITS header in the JWST routines. [#174]
+
 ramp_fitting
 ~~~~~~~~~~~~
 
 - Changing where time division occurs during ramp fitting in order to
   properly handle special cases where the time is not group time, such
   as when ZEROFRAME data is used, so the time is frame time. [#173]
 
+- Added another line of code to be included in the section where warnings are turned
+  off. The large number of warnings can cause a hang in the Jupyter notebook when
+  running with multiprocessing. [#174]
+
 Changes to API
 --------------
 

diff --git a/src/stcal/jump/jump.py b/src/stcal/jump/jump.py
diff --git a/src/stcal/jump/twopoint_difference.py b/src/stcal/jump/twopoint_difference.py
diff --git a/src/stcal/ramp_fitting/ols_fit.py b/src/stcal/ramp_fitting/ols_fit.py
@@ -1125,11 +1125,11 @@ def ramp_fit_compute_variances(ramp_data, gain_2d, readnoise_2d, fit_slopes_ans)
         #   variance is proportional to the median estimated slope) to
         #   outrageously large values so that they will have negligible
         #   contributions.
-        var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
 
         # Suppress, then re-enable harmless arithmetic warnings
         warnings.filterwarnings("ignore", ".*invalid value.*", RuntimeWarning)
         warnings.filterwarnings("ignore", ".*divide by zero.*", RuntimeWarning)
+        var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
         var_p4[var_p4 <= 0.] = utils.LARGE_VARIANCE
 
         var_r4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)

diff --git a/src/stcal/saturation/saturation.py b/src/stcal/saturation/saturation.py
@@ -61,7 +61,6 @@ def flag_saturated_pixels(
     """
 
     nints, ngroups, nrows, ncols = data.shape
-
     saturated = dqflags['SATURATED']
     ad_floor = dqflags['AD_FLOOR']
     no_sat_check = dqflags['NO_SAT_CHECK']

diff --git a/tests/test_jump.py b/tests/test_jump.py
@@ -155,7 +155,7 @@ def test_find_faint_extended():
     rng = np.random.default_rng(12345)
     data[0, 1:, 14:20, 15:20] = 6 * gain * 1.7
     data = data + rng.normal(size=(nint, ngrps, nrows, ncols)) * readnoise
-    gdq, num_showers = find_faint_extended(data, gdq, readnoise, 1,
+    gdq, num_showers = find_faint_extended(data, gdq, readnoise, 1, 100,
                                            snr_threshold=1.3,
                                            min_shower_area=20, inner=1,
                                            outer=2, sat_flag=2, jump_flag=4,
@@ -193,6 +193,40 @@ def test_find_faint_extended():
     #  Check that the flags are not applied in the 3rd group after the event
     assert (np.all(gdq[0, 4, 12:22, 14:23]) == 0)
 
+# No shower is found because the event is identical in all ints
+def test_find_faint_extended_sigclip():
+    nint, ngrps, ncols, nrows = 101, 6, 30, 30
+    data = np.zeros(shape=(nint, ngrps, nrows, ncols), dtype=np.float32)
+    gdq = np.zeros_like(data, dtype=np.uint8)
+    gain = 4
+    readnoise = np.ones(shape=(nrows, ncols), dtype=np.float32) * 6.0 * gain
+    rng = np.random.default_rng(12345)
+    data[0, 1:, 14:20, 15:20] = 6 * gain * 1.7
+    data = data + rng.normal(size=(nint, ngrps, nrows, ncols)) * readnoise
+    gdq, num_showers = find_faint_extended(data, gdq, readnoise, 1, 100,
+                                           snr_threshold=1.3,
+                                           min_shower_area=20, inner=1,
+                                           outer=2, sat_flag=2, jump_flag=4,
+                                           ellipse_expand=1.1, num_grps_masked=3)
+    #  Check that all the expected samples in group 2 are flagged as jump and
+    #  that they are not flagged outside
+    assert (np.all(gdq[0, 1, 22, 14:23] == 0))
+    assert (np.all(gdq[0, 1, 21, 16:20] == 0))
+    assert (np.all(gdq[0, 1, 20, 15:22] == 0))
+    assert (np.all(gdq[0, 1, 19, 15:23] == 0))
+    assert (np.all(gdq[0, 1, 18, 14:23] == 0))
+    assert (np.all(gdq[0, 1, 17, 14:23] == 0))
+    assert (np.all(gdq[0, 1, 16, 14:23] == 0))
+    assert (np.all(gdq[0, 1, 15, 14:22] == 0))
+    assert (np.all(gdq[0, 1, 14, 16:22] == 0))
+    assert (np.all(gdq[0, 1, 13, 17:21] == 0))
+    assert (np.all(gdq[0, 1, 12, 14:23] == 0))
+    assert (np.all(gdq[0, 1, 12:23, 24] == 0))
+    assert (np.all(gdq[0, 1, 12:23, 13] == 0))
+
+    #  Check that the flags are not applied in the 3rd group after the event
+    assert (np.all(gdq[0, 4, 12:22, 14:23]) == 0)
+
 
 def test_inside_ellipse5():
     ellipse = ((0, 0), (1, 2), -10)

diff --git a/tests/test_ramp_fitting.py b/tests/test_ramp_fitting.py
@@ -1,5 +1,4 @@
 import numpy as np
-
 from stcal.ramp_fitting.ramp_fit import ramp_fit_data
 from stcal.ramp_fitting.ramp_fit_class import RampData