Changes to Offset Function (#207)

* change default offset_min_skybrightness * change default offset_min_skybrightness in actor * safety_factor should be None by default (set within coordio) * return offset_flags from object_offset * check offset flags to see where valid * in mag array 0 index should be sdss g, not gaia_g * Keep robots with invalid offsets folded * Update changelog * Ensure the ToO df has the same shape as the target df * correctly check if all flags are passed * Add new offset flags to TARGET_DATA_SCHEMA * Fix call to numpy and reformat * Use infer_schema_length=1500 to ensure types * Add warning for invalid offsets * correct checking of offset flags * np -> numpy and format code --------- Co-authored-by: José Sánchez-Gallego <[email protected]>
sdss · Oct 24, 2024 · a6a1992 · a6a1992
1 parent 06b3821
commit a6a1992
Show file tree

Hide file tree

Showing 6 changed files with 70 additions and 22 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,8 +8,10 @@
 
 ### ✨ Improved
 
+* [#207](https://github.com/sdss/jaeger/pull/207) Update the call to determine target offsets.
 * [#206](https://github.com/sdss/jaeger/pull/206) Added flag `--sea-anemone` to `jaeger configuration random`.
 
+
 ### ⚙️ Engineering
 
 * Relax Python requirement to `^3.10,<4`.

diff --git a/src/jaeger/actor/commands/configuration.py b/src/jaeger/actor/commands/configuration.py
@@ -62,7 +62,7 @@ async def _load_design(
     apogee_wavelength: float | None = None,
     get_paths: bool = True,
     path_generation_mode: str | None = None,
-    safety_factor: float = 1,
+    safety_factor: float | None = None,
     offset_min_skybrightness: float = 0.5,
 ):
     """Helper to load or preload a design."""
@@ -374,7 +374,7 @@ async def load(
     no_clone: bool = False,
     path_generation_mode: str | None = None,
     safety_factor: float = 1,
-    offset_min_skybrightness: float = 0.5,
+    offset_min_skybrightness: float = 0.0,
 ):
     """Creates and ingests a configuration from a design in the database."""
 

diff --git a/src/jaeger/target/assignment.py b/src/jaeger/target/assignment.py
@@ -178,26 +178,30 @@ def create_fibre_data(self):
                     if len(tdata_row) == 1:
                         tdata_row_dict = tdata_row.rows(named=True)[0]
 
-                        hole_data.update(
-                            {
-                                "catalogid": tdata_row_dict["catalogid"],
-                                "ra_icrs": tdata_row_dict["ra"],
-                                "dec_icrs": tdata_row_dict["dec"],
-                                "pmra": tdata_row_dict["pmra"],
-                                "pmdec": tdata_row_dict["pmdec"],
-                                "parallax": tdata_row_dict["parallax"],
-                                "coord_epoch": tdata_row_dict["epoch"],
-                                "delta_ra": tdata_row_dict["delta_ra"],
-                                "delta_dec": tdata_row_dict["delta_dec"],
-                                "assigned": True,
-                                "too": tdata_row_dict["is_too"],
-                            }
-                        )
+                        # Keep robots with invalid offsets folded.
+                        offset_valid = tdata_row_dict.get("offset_valid", True)
+
+                        if offset_valid:
+                            hole_data.update(
+                                {
+                                    "catalogid": tdata_row_dict["catalogid"],
+                                    "ra_icrs": tdata_row_dict["ra"],
+                                    "dec_icrs": tdata_row_dict["dec"],
+                                    "pmra": tdata_row_dict["pmra"],
+                                    "pmdec": tdata_row_dict["pmdec"],
+                                    "parallax": tdata_row_dict["parallax"],
+                                    "coord_epoch": tdata_row_dict["epoch"],
+                                    "delta_ra": tdata_row_dict["delta_ra"],
+                                    "delta_dec": tdata_row_dict["delta_dec"],
+                                    "assigned": True,
+                                    "too": tdata_row_dict["is_too"],
+                                }
+                            )
 
                 fibre_tdata.append(hole_data)
 
         # Create initial DF from wok_data. This contains 1500 columns, one per fibre.
-        fibre_data = polars.DataFrame(fibre_tdata)
+        fibre_data = polars.DataFrame(fibre_tdata, infer_schema_length=1500)
 
         # Add empty columns for the rest of the schema. Negate boolean columns.
         fibre_data = (

diff --git a/src/jaeger/target/design.py b/src/jaeger/target/design.py
@@ -81,8 +81,8 @@ def __init__(
         create_configuration: bool = True,
         epoch: float | None = None,
         scale: float | None = None,
-        safety_factor: float = 0.1,
-        offset_min_skybrightness: float = 0.5,
+        safety_factor: float | None = None,
+        offset_min_skybrightness: float = 0.0,
         use_targets_of_opportunity: bool = True,
     ):
         if calibration.wokCoords is None:
@@ -214,7 +214,7 @@ def _offset(group: polars.DataFrame):
 
             mag = numpy.array(
                 [
-                    group["gaia_g"].to_numpy(),
+                    group["g"].to_numpy(),
                     group["r"].to_numpy(),
                     group["i"].to_numpy(),
                     group["z"].to_numpy(),
@@ -247,7 +247,7 @@ def _offset(group: polars.DataFrame):
                 # object_offset that will return delta_ra=-1 when the design_mode
                 # doesn't have any magnitude limits defined.
 
-                delta_ra, delta_dec, _ = object_offset(
+                delta_ra, delta_dec, offset_flags = object_offset(
                     mag,
                     numpy.array(mag_lim),
                     lunation,
@@ -261,18 +261,56 @@ def _offset(group: polars.DataFrame):
             else:
                 delta_ra = numpy.zeros(len(group))
                 delta_dec = numpy.zeros(len(group))
+                offset_flags = numpy.zeros(len(group), dtype=numpy.int32)
+
+            # make bad mag cases nan
+            cases = [-999, -9999, 999, 0.0, numpy.nan, 99.9, None]
+            mag[numpy.isin(mag, cases)] = numpy.nan
+
+            # check stars that are too bright for design mode
+            mag_lim = numpy.array(mag_lim)
+            valid_ind = numpy.where(numpy.array(mag_lim) != -999.0)[0]
+            mag_bright = numpy.any(mag[:, valid_ind] < mag_lim[valid_ind], axis=1)
+
+            # grab program as below check not valid for skies or standards
+            program = group["program"].to_numpy()
+
+            # check offset flags to see if should be used or not
+            offset_valid = numpy.zeros(len(group), dtype=bool)
+            for i, fl in enumerate(offset_flags):
+                # manually check bad flags
+                if program[i] == "SKY" or "ops" in program[i]:
+                    offset_valid[i] = True
+                elif 8 & int(fl) and mag_bright[i]:
+                    # if below sky brightness and brighter than mag limit
+                    offset_valid[i] = False
+                elif 16 & int(fl) and mag_bright[i]:
+                    # if can_offset False and brighter than mag limit
+                    offset_valid[i] = False
+                elif 32 & int(fl):
+                    # if brighter than safety limit
+                    offset_valid[i] = False
+                else:
+                    offset_valid[i] = True
 
             assert isinstance(delta_ra, numpy.ndarray)
             assert isinstance(delta_dec, numpy.ndarray)
+            assert isinstance(offset_flags, numpy.ndarray)
 
             return group.with_columns(
                 delta_ra=polars.Series(values=delta_ra, dtype=polars.Float32),
                 delta_dec=polars.Series(values=delta_dec, dtype=polars.Float32),
+                offset_flags=polars.Series(values=offset_flags, dtype=polars.Int32),
+                offset_valid=polars.Series(values=offset_valid, dtype=polars.Boolean),
             )
 
         log.debug(f"offset_min_skybrightness={self.offset_min_skybrightness}")
         log.debug(f"safety_factor={self.safety_factor}")
 
+        invalid = target_data.filter(~polars.col.offset_valid)
+        if len(invalid) > 0:
+            log.warning(f"Found {len(invalid)} targets with invalid offsets.")
+
         # Group by fibre type and apply the offset calculation. delta_ra and delta_dec
         # are modified and target_data is updated.
         target_data = target_data.group_by("fibre_type").map_groups(_offset)

diff --git a/src/jaeger/target/schemas.py b/src/jaeger/target/schemas.py
@@ -180,6 +180,8 @@
     "lambda_eff": polars.Float32,
     "delta_ra": polars.Float32,
     "delta_dec": polars.Float32,
+    "offset_flags": polars.Int32,
+    "offset_valid": polars.Boolean,
     "can_offset": polars.Boolean,
     "priority": polars.Int32,
     "catalogid": polars.Int64,

diff --git a/src/jaeger/target/too.py b/src/jaeger/target/too.py
@@ -158,6 +158,8 @@ def add_targets_of_opportunity_to_design(design: Design):
                 "epoch": too_entry["epoch"],
                 "delta_ra": too_entry["delta_ra"],
                 "delta_dec": too_entry["delta_dec"],
+                "offset_flags": 0,
+                "offset_valid": True,
                 "can_offset": too_entry["can_offset"],
                 "lambda_eff": defaults.INST_TO_WAVE[fibre_type.capitalize()],
                 "g": too_entry["g_mag"],