Merge pull request #1023 from nasa/issue_1021_disp_s1_v3.0.0_integration

DISP-S1 PGE v3.0.0 Integration

cluster_provisioning/dev-e2e/variables.tf

@@ -389,7 +389,7 @@ variable "pge_releases" {
     "cslc_s1"  = "2.1.1"
     "rtc_s1"   = "2.1.1"
     "dswx_s1"  = "3.0.2"
-    "disp_s1"  = "3.0.0-rc.4.2"
+    "disp_s1"  = "3.0.0"
     "dswx_ni"  = "4.0.0-er.3.0"
   }
 }
@@ -513,7 +513,7 @@ variable "amis" {
     mozart    = "ami-056fd00b4ff240ccd" # mozart v4.26 - 241015
     metrics   = "ami-003eba97ad98a4ce2" # metrics v4.18 - 241015
     grq       = "ami-09de0d30ecebcb202" # grq v4.19 - 241015
-    factotum  = "ami-019ba050c9196bc95" # factotum v4.17 - 241015 
+    factotum  = "ami-019ba050c9196bc95" # factotum v4.17 - 241015
     autoscale = "ami-0bb7ad9edded1a1b7" # verdi v4.17 patchdate - 241015
   }
 }

cluster_provisioning/dev-int/override.tf

@@ -176,7 +176,7 @@ variable "pge_releases" {
     "cslc_s1"  = "2.1.1"
     "rtc_s1"   = "2.1.1"
     "dswx_s1"  = "3.0.2"
-    "disp_s1"  = "3.0.0-rc.4.2"
+    "disp_s1"  = "3.0.0"
     "dswx_ni"  = "4.0.0-er.3.0"
   }
 }

cluster_provisioning/dev/variables.tf

@@ -390,7 +390,7 @@ variable "pge_releases" {
     "cslc_s1"  = "2.1.1"
     "rtc_s1"   = "2.1.1"
     "dswx_s1"  = "3.0.2"
-    "disp_s1"  = "3.0.0-rc.4.2"
+    "disp_s1"  = "3.0.0"
     "dswx_ni"  = "4.0.0-er.3.0"
   }
 }

cluster_provisioning/ebs-snapshot/variables.tf

@@ -34,7 +34,7 @@ variable "pge_releases" {
     "cslc_s1"  = "2.1.1"
     "rtc_s1"   = "2.1.1"
     "dswx_s1"  = "3.0.2"
-    "disp_s1"  = "3.0.0-rc.4.2"
+    "disp_s1"  = "3.0.0"
     "dswx_ni"  = "4.0.0-er.3.0"
   }
 }

cluster_provisioning/modules/common/variables.tf

@@ -649,7 +649,7 @@ variable "pge_releases" {
     "cslc_s1"  = "2.1.1"
     "rtc_s1"   = "2.1.1"
     "dswx_s1"  = "3.0.2"
-    "disp_s1"  = "3.0.0-rc.4.2"
+    "disp_s1"  = "3.0.0"
     "dswx_ni"  = "4.0.0-er.3.0"
   }
 }

conf/RunConfig.yaml.L3_DISP_S1.jinja2.tmpl

@@ -36,6 +36,7 @@ RunConfig:
         SchemaPath: /home/mamba/opera/pge/disp_s1/schema/disp_s1_sas_schema.yaml
         AlgorithmParametersSchemaPath: /home/mamba/opera/pge/disp_s1/schema/algorithm_parameters_disp_s1_schema.yaml
         IsoTemplatePath: /home/mamba/opera/pge/disp_s1/templates/OPERA_ISO_metadata_L3_DISP_S1_template.xml.jinja2
+        IsoMeasuredParameterDescriptions: /home/mamba/opera/pge/disp_s1/templates/disp_s1_measured_parameters.yaml
       QAExecutable:
         Enabled: False
         ProgramPath:

conf/schema/RunConfig_schema.L3_DISP_S1.yaml

@@ -26,6 +26,7 @@ RunConfig:
         SchemaPath: str(required=True)
         AlgorithmParametersSchemaPath: str(required=False)
         IsoTemplatePath: str(required=False)
+        IsoMeasuredParameterDescriptions: str(required=False)
         DataValidityStartDate: int(min=20000101, max=21991231, required=False)
 
       QAExecutable:
@@ -76,10 +77,6 @@ RunConfig:
         # JSON file containing list of reference date changes for each frame
         reference_date_database_json: str(required=False)
 
-        # JSON file containing frame-specific algorithm parameters to override the
-        # defaults passed in the `algorithm_parameters.yaml`.
-        algorithm_parameters_overrides_json: str(required=False)
-
       primary_executable:
         # Product type of the PGE.
         product_type: enum('DISP_S1_FORWARD', 'DISP_S1_HISTORICAL')

conf/settings.yaml

@@ -26,7 +26,7 @@ RTC_S1_STATIC_PRODUCT_VERSION: "1.0"
 DSWX_S1_COLLECTION_NAME: "OPERA_L3_DSWX-S1_V1"
 DSWX_S1_PRODUCT_VERSION: "1.0"
 DISP_S1_COLLECTION_NAME: "OPERA_L3_DISP-S1_PROVISIONAL_V0"
-DISP_S1_PRODUCT_VERSION: "0.7"
+DISP_S1_PRODUCT_VERSION: "0.8"
 
 # DSWx-NI in R4
 DSWX_NI_COLLECTION_NAME: "OPERA_L3_DSWX-NI_PROVISIONAL_V0"

docker/job-spec.json.SCIFLO_L3_DISP_S1

@@ -10,8 +10,8 @@
   },
   "dependency_images": [
     {
-      "container_image_name": "opera_pge/disp_s1:3.0.0-rc.4.2",
-      "container_image_url": "$CODE_BUCKET_URL/opera_pge-disp_s1-3.0.0-rc.4.2.tar.gz",
+      "container_image_name": "opera_pge/disp_s1:3.0.0",
+      "container_image_url": "$CODE_BUCKET_URL/opera_pge-disp_s1-3.0.0.tar.gz",
       "container_mappings": {
         "$HOME/.netrc": ["/root/.netrc"],
         "$HOME/.aws": ["/root/.aws", "ro"]

docker/job-spec.json.SCIFLO_L3_DISP_S1_hist

@@ -10,8 +10,8 @@
   },
   "dependency_images": [
     {
-      "container_image_name": "opera_pge/disp_s1:3.0.0-rc.4.2",
-      "container_image_url": "$CODE_BUCKET_URL/opera_pge-disp_s1-3.0.0-rc.4.2.tar.gz",
+      "container_image_name": "opera_pge/disp_s1:3.0.0",
+      "container_image_url": "$CODE_BUCKET_URL/opera_pge-disp_s1-3.0.0.tar.gz",
       "container_mappings": {
         "$HOME/.netrc": ["/root/.netrc"],
         "$HOME/.aws": ["/root/.aws", "ro"]

docker/job-spec.json.pge_smoke_test

@@ -18,8 +18,8 @@
           }
         },
         {
-          "container_image_name": "opera_pge/disp_s1:3.0.0-rc.4.2",
-          "container_image_url": "$CODE_BUCKET_URL/opera_pge-disp_s1-3.0.0-rc.4.2.tar.gz",
+          "container_image_name": "opera_pge/disp_s1:3.0.0",
+          "container_image_url": "$CODE_BUCKET_URL/opera_pge-disp_s1-3.0.0.tar.gz",
           "container_mappings": {
             "$HOME/.netrc": ["/root/.netrc"],
             "$HOME/.aws": ["/root/.aws", "ro"]

opera_chimera/configs/pge_configs/PGE_L3_DISP_S1.yaml

@@ -24,7 +24,6 @@ runconfig:
   static_ancillary_file_group:
     frame_to_burst_json: __CHIMERA_VAL__
     reference_date_database_json: __CHIMERA_VAL__
-    algorithm_parameters_overrides_json: __CHIMERA_VAL__
   product_path_group:
     product_version: __CHIMERA_VAL__
     product_path: /home/mamba/output_dir
@@ -33,10 +32,11 @@ runconfig:
   processing:
     frame_id: __CHIMERA_VAL__
     algorithm_parameters: __CHIMERA_VAL__
+    algorithm_parameters_overrides_json: __CHIMERA_VAL__
     product_type: __CHIMERA_VAL__
     threads_per_worker: __CHIMERA_VAL__
     n_parallel_bursts: __CHIMERA_VAL__
-    debug_switch: True  # TODO: Currently enabled to bypass a faulty validation check in the PGE
+    debug_switch: False
   cnm_version: "__CHIMERA_VAL__"
 
 # This lists all the precondition evaluation steps that this PGE needs to run prior to running the PGE.
@@ -79,7 +79,8 @@ get_static_ancillary_files:
     s3_key: "disp_frames/disp_s1_frame_reference_dates/opera-disp-s1-reference-dates-minimal-2024-09-19.json"
   algorithm_parameters_overrides_json:
     s3_bucket: "opera-ancillaries"
-    s3_key: "algorithm_parameters/disp_s1/0.4.6/opera-disp-s1-algorithm-parameters-overrides-2024-10-14.json"
+    s3_key: "algorithm_parameters/disp_s1/0.5.1/opera-disp-s1-algorithm-parameters-overrides-2024-10-14.json"
+    download: true
 
 get_disp_s1_mask_file:
   s3_bucket: "opera-water-mask"
@@ -89,7 +90,7 @@ get_disp_s1_algorithm_parameters:
   # This S3 path only defines a location pattern of an algorithm parameters template file,
   # the processing mode (forward vs. historical) will be filled in by the function itself
   s3_bucket: "opera-ancillaries"
-  s3_key: "algorithm_parameters/disp_s1/0.4.6/algorithm_parameters_{processing_mode}.yaml.tmpl"
+  s3_key: "algorithm_parameters/disp_s1/0.5.1/algorithm_parameters_{processing_mode}.yaml.tmpl"
 
 instantiate_algorithm_parameters_template:
   template_mapping:

opera_chimera/precondition_functions.py

@@ -1518,15 +1518,24 @@ def get_static_ancillary_files(self):
         """
         logger.info(f"Evaluating precondition {inspect.currentframe().f_code.co_name}")
 
+        working_dir = get_working_dir()
+
         rc_params = {}
 
         static_ancillary_products = self._pge_config.get(oc_const.GET_STATIC_ANCILLARY_FILES, {})
 
         for static_ancillary_product in static_ancillary_products.keys():
             s3_bucket = static_ancillary_products.get(static_ancillary_product, {}).get(oc_const.S3_BUCKET)
             s3_key = static_ancillary_products.get(static_ancillary_product, {}).get(oc_const.S3_KEY)
+            download = static_ancillary_products.get(static_ancillary_product, {}).get("download", False)
 
-            rc_params[static_ancillary_product] = f"s3://{s3_bucket}/{s3_key}"
+            if download:
+                output_filepath = os.path.join(working_dir, os.path.basename(s3_key))
+                pge_metrics = download_object_from_s3(s3_bucket, s3_key, output_filepath)
+                write_pge_metrics(os.path.join(working_dir, "pge_metrics.json"), pge_metrics)
+                rc_params[static_ancillary_product] = output_filepath
+            else:
+                rc_params[static_ancillary_product] = f"s3://{s3_bucket}/{s3_key}"
 
         logger.info(f"rc_params : {rc_params}")
 

tests/unit/opera_chimera/test_precondition_functions.py

@@ -32,6 +32,7 @@ class MockGdal:
     def UseExceptions(*args):
         pass
 
+
     @staticmethod
     def Translate(destName, srcDS, **kwargs):
         with open(destName, 'w') as outfile:
@@ -56,12 +57,33 @@ def __init__(self):
 
             return MockRasterBand()
 
+        def GetProjection(self):
+            pass
+
     @staticmethod
     def Open(filename, filemode=None):
         """Mock implementation for gdal.Open. Returns an instance of the mock Dataset."""
         return MockGdal.MockGdalDataset()
 
 
+class MockOsr:
+    """Mock class for the osgeo.osr module."""
+
+    class MockSpatialReference:
+        """Mock class for the osgeo.osr module"""
+
+        def __init__(self, *args, **kwargs):
+            self.zone = 1
+            self.hemi = 'N'
+
+        def GetAttrValue(self, *args):
+            return '4326'
+
+    @staticmethod
+    def SpatialReference(*args, **kwargs):
+        """Mock implementation for osgeo.osr.SpatialReference"""
+        return MockOsr.MockSpatialReference(*args, **kwargs)
+
 def _check_aws_connection_patch(bucket, key):
     """
     No-op patch function for use with testing precondition functions that attempt

tools/stage_ancillary_map.py

@@ -7,7 +7,7 @@
 
 import backoff
 
-from osgeo import gdal
+from osgeo import gdal, osr
 
 from commons.logger import logger
 from commons.logger import LogLevels
@@ -96,6 +96,24 @@ def download_map(polys, map_bucket, map_vrt_key, outfile):
             output_path, ds, format='GTiff', projWin=[x_min, y_max, x_max, y_min]
         )
 
+        # stage_ancillary_map.py takes a bbox as an input. The longitude coordinates
+        # of this bbox are unwrapped i.e., range in [0, 360] deg. If the
+        # bbox crosses the anti-meridian, the script divides it in two
+        # bboxes neighboring the anti-meridian. Here, x_min and x_max
+        # represent the min and max longitude coordinates of one of these
+        # bboxes. We Add 360 deg if the min longitude of the downloaded DEM
+        # tile is < 180 deg i.e., there is a dateline crossing.
+        # This ensures that the mosaicked DEM VRT will span a min
+        # range of longitudes rather than the full [-180, 180] deg
+        sr = osr.SpatialReference(ds.GetProjection())
+        epsg_str = sr.GetAttrValue("AUTHORITY", 1)
+
+        if x_min <= -180.0 and epsg_str == '4326':
+            ds = gdal.Open(output_path, gdal.GA_Update)
+            geotransform = list(ds.GetGeoTransform())
+            geotransform[0] += 360.0
+            ds.SetGeoTransform(tuple(geotransform))
+
     # Build VRT with downloaded sub-regions
     gdal.BuildVRT(outfile, region_list)