From aa929d78c5a7c86dc67144a19d5141312f98e681 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Tue, 17 Dec 2024 22:00:22 +0100 Subject: [PATCH] fixup! Issue #346 Some more ProcessArgs porting --- openeo_driver/ProcessGraphDeserializer.py | 130 +++++++++++----------- openeo_driver/processes.py | 10 +- tests/test_processes.py | 9 ++ 3 files changed, 82 insertions(+), 67 deletions(-) diff --git a/openeo_driver/ProcessGraphDeserializer.py b/openeo_driver/ProcessGraphDeserializer.py index c505094a..0a406f18 100644 --- a/openeo_driver/ProcessGraphDeserializer.py +++ b/openeo_driver/ProcessGraphDeserializer.py @@ -754,7 +754,7 @@ def load_disk_data(args: ProcessArgs, env: EvalEnv) -> DriverDataCube: """ Deprecated, use load_uploaded_files or load_stac """ - _log.warning("Deprecated: usage of load_disk_data") + _log.warning("DEPRECATED: load_disk_data usage") kwargs = dict( glob_pattern=args.get_required("glob_pattern", expected_type=str), format=args.get_required("format", expected_type=str), @@ -1002,7 +1002,9 @@ def apply_polygon(args: ProcessArgs, env: EvalEnv) -> DriverDataCube: process = args.get_deep("process", "process_graph", expected_type=dict) if "polygons" in args and "geometries" not in args: # TODO remove this deprecated "polygons" parameter handling when not used anymore - _log.warning("In process 'apply_polygon': parameter 'polygons' is deprecated, use 'geometries' instead.") + _log.warning( + "DEPRECATED: In process 'apply_polygon': parameter 'polygons' is deprecated, use 'geometries' instead." + ) geometries = args.get_required("polygons") else: geometries = args.get_required("geometries") @@ -1528,16 +1530,30 @@ def resample_spatial(args: ProcessArgs, env: EvalEnv) -> DriverDataCube: @process -def resample_cube_spatial(args: dict, env: EvalEnv) -> DriverDataCube: - image_collection = extract_arg(args, 'data') - target_image_collection = extract_arg(args, 'target') - method = args.get('method', 'near') - if not isinstance(image_collection, DriverDataCube): - raise ProcessParameterInvalidException( - parameter="data", process="resample_cube_spatial", - reason=f"Invalid data type {type(image_collection)!r} expected raster-cube." - ) - return image_collection.resample_cube_spatial(target=target_image_collection, method=method) +def resample_cube_spatial(args: ProcessArgs, env: EvalEnv) -> DriverDataCube: + cube: DriverDataCube = args.get_required("data", expected_type=DriverDataCube) + target: DriverDataCube = args.get_required("target", expected_type=DriverDataCube) + method = args.get_enum( + "method", + options=[ + "average", + "bilinear", + "cubic", + "cubicspline", + "lanczos", + "max", + "med", + "min", + "mode", + "near", + "q1", + "q3", + "rms", + "sum", + ], + default="near", + ) + return cube.resample_cube_spatial(target=target, method=method) @process @@ -1636,25 +1652,22 @@ def run_udf(args: dict, env: EvalEnv): @process -def linear_scale_range(args: dict, env: EvalEnv) -> DriverDataCube: - image_collection = extract_arg(args, 'x') - - inputMin = extract_arg(args, "inputMin") - inputMax = extract_arg(args, "inputMax") - outputMax = args.get("outputMax", 1.0) - outputMin = args.get("outputMin", 0.0) - if not isinstance(image_collection, DriverDataCube): - raise ProcessParameterInvalidException( - parameter="data", process="linear_scale_range", - reason=f"Invalid data type {type(image_collection)!r} expected raster-cube." - ) - - return image_collection.linear_scale_range(inputMin, inputMax, outputMin, outputMax) +def linear_scale_range(args: ProcessArgs, env: EvalEnv) -> DriverDataCube: + # TODO: eliminate this top-level linear_scale_range process implementation (should be used as `apply` callback) + _log.warning("DEPRECATED: linear_scale_range usage directly on cube is deprecated/non-standard.") + cube: DriverDataCube = args.get_required("x", expected_type=DriverDataCube) + # Note: non-standard camelCase parameter names (https://github.com/Open-EO/openeo-processes/issues/302) + input_min = args.get_required("inputMin") + input_max = args.get_required("inputMax") + output_min = args.get_optional("outputMin", default=0.0) + output_max = args.get_optional("outputMax", default=1.0) + # TODO linear_scale_range is defined on GeopysparkDataCube, but not on DriverDataCube + return cube.linear_scale_range(input_min, input_max, output_min, output_max) @process -def constant(args: dict, env: EvalEnv): - return args["x"] +def constant(args: ProcessArgs, env: EvalEnv): + return args.get_required("x") def flatten_children_node_types(process_graph: Union[dict, list]): @@ -1806,10 +1819,11 @@ def apply_process(process_id: str, args: dict, namespace: Union[str, None], env: ]) .returns("GeoJSON-style feature collection", schema={"type": "object", "subtype": "geojson"}) ) -def read_vector(args: Dict, env: EvalEnv) -> DelayedVector: +def read_vector(args: ProcessArgs, env: EvalEnv) -> DelayedVector: # TODO #114 EP-3981: deprecated in favor of load_uploaded_files/load_external? https://github.com/Open-EO/openeo-processes/issues/322 # TODO: better argument name than `filename`? - path = extract_arg(args, "filename") + _log.warning("DEPRECATED: read_vector usage") + path = args.get_required("filename") _check_geometry_path_assumption( path=path, process="read_vector", parameter="filename" ) @@ -1856,10 +1870,10 @@ def load_uploaded_files(args: ProcessArgs, env: EvalEnv) -> Union[DriverVectorCu .param('data', description="GeoJson object.", schema={"type": "object", "subtype": "geojson"}) .returns("vector-cube", schema={"type": "object", "subtype": "vector-cube"}) ) -def to_vector_cube(args: Dict, env: EvalEnv): - _log.warning("Experimental process `to_vector_cube` is deprecated, use `load_geojson` instead") +def to_vector_cube(args: ProcessArgs, env: EvalEnv): + _log.warning("DEPRECATED: process to_vector_cube is deprecated, use load_geojson instead") # TODO: remove this experimental/deprecated process - data = extract_arg(args, "data", process_id="to_vector_cube") + data = args.get_required("data") if isinstance(data, dict) and data.get("type") in {"Polygon", "MultiPolygon", "Feature", "FeatureCollection"}: return env.backend_implementation.vector_cube_cls.from_geojson(data) raise FeatureUnsupportedException(f"Converting {type(data)} to vector cube is not supported") @@ -1925,14 +1939,10 @@ def get_geometries(args: Dict, env: EvalEnv) -> Union[DelayedVector, dict]: .param('data', description="A raster data cube.", schema={"type": "object", "subtype": "raster-cube"}) .returns("vector-cube", schema={"type": "object", "subtype": "vector-cube"}) ) -def raster_to_vector(args: Dict, env: EvalEnv): - image_collection = extract_arg(args, 'data') - if not isinstance(image_collection, DriverDataCube): - raise ProcessParameterInvalidException( - parameter="data", process="raster_to_vector", - reason=f"Invalid data type {type(image_collection)!r} expected raster-cube." - ) - return image_collection.raster_to_vector() +def raster_to_vector(args: ProcessArgs, env: EvalEnv): + cube: DriverDataCube = args.get_required("data", expected_type=DriverDataCube) + # TODO: raster_to_vector is only defined on GeopysparkDataCube, not DriverDataCube + return cube.raster_to_vector() @non_standard_process( @@ -2056,9 +2066,9 @@ def evaluate_process_from_url(process_id: str, namespace: str, args: dict, env: .param('seconds', description="Number of seconds to sleep.", schema={"type": "number"}, required=True) .returns("Original data", schema={}) ) -def sleep(args: Dict, env: EvalEnv): - data = extract_arg(args, "data") - seconds = extract_arg(args, "seconds") +def sleep(args: ProcessArgs, env: EvalEnv): + data = args.get_required("data") + seconds = args.get_required("seconds", expected_type=(int, float)) dry_run_tracer: DryRunDataTracer = env.get(ENV_DRY_RUN_TRACER) if not dry_run_tracer: _log.info("Sleeping {s} seconds".format(s=seconds)) @@ -2165,20 +2175,15 @@ def resolution_merge(args: ProcessArgs, env: EvalEnv): .param('data', description="Data to discard.", schema={}, required=False) .returns("Nothing", schema={}) ) -def discard_result(args: Dict, env: EvalEnv): +def discard_result(args: ProcessArgs, env: EvalEnv): # TODO: keep a reference to the discarded result? return NullResult() @process_registry_100.add_function(spec=read_spec("openeo-processes/experimental/mask_scl_dilation.json")) @process_registry_2xx.add_function(spec=read_spec("openeo-processes/experimental/mask_scl_dilation.json")) -def mask_scl_dilation(args: Dict, env: EvalEnv): - cube: DriverDataCube = extract_arg(args, 'data') - if not isinstance(cube, DriverDataCube): - raise ProcessParameterInvalidException( - parameter="data", process="mask_scl_dilation", - reason=f"Invalid data type {type(cube)!r} expected raster-cube." - ) +def mask_scl_dilation(args: ProcessArgs, env: EvalEnv): + cube: DriverDataCube = args.get_required("data", expected_type=DriverDataCube) if hasattr(cube, "mask_scl_dilation"): the_args = args.copy() del the_args["data"] @@ -2209,13 +2214,8 @@ def to_scl_dilation_mask(args: ProcessArgs, env: EvalEnv): @process_registry_100.add_function(spec=read_spec("openeo-processes/experimental/mask_l1c.json")) @process_registry_2xx.add_function(spec=read_spec("openeo-processes/experimental/mask_l1c.json")) -def mask_l1c(args: Dict, env: EvalEnv): - cube: DriverDataCube = extract_arg(args, 'data') - if not isinstance(cube, DriverDataCube): - raise ProcessParameterInvalidException( - parameter="data", process="mask_l1c", - reason=f"Invalid data type {type(cube)!r} expected raster-cube." - ) +def mask_l1c(args: ProcessArgs, env: EvalEnv): + cube: DriverDataCube = args.get_required("data", expected_type=DriverDataCube) if hasattr(cube, "mask_l1c"): return cube.mask_l1c() else: @@ -2280,8 +2280,8 @@ def array_create(args: ProcessArgs, env: EvalEnv) -> list: @process_registry_100.add_function(spec=read_spec("openeo-processes/1.x/proposals/load_result.json")) -def load_result(args: dict, env: EvalEnv) -> DriverDataCube: - job_id = extract_arg(args, "id") +def load_result(args: ProcessArgs, env: EvalEnv) -> DriverDataCube: + job_id = args.get_required("id", expected_type=str) user = env.get("user") arguments = {} @@ -2309,10 +2309,10 @@ def load_result(args: dict, env: EvalEnv) -> DriverDataCube: @process_registry_100.add_function(spec=read_spec("openeo-processes/1.x/proposals/inspect.json")) @process_registry_2xx.add_function(spec=read_spec("openeo-processes/2.x/proposals/inspect.json")) -def inspect(args: dict, env: EvalEnv): - data = extract_arg(args, "data") - message = args.get("message", "") - level = args.get("level", "info") +def inspect(args: ProcessArgs, env: EvalEnv): + data = args.get_required("data") + message = args.get_optional("message", default="") + level = args.get_optional("level", default="info") if message: _log.log(level=logging.getLevelName(level.upper()), msg=message) data_message = str(data) diff --git a/openeo_driver/processes.py b/openeo_driver/processes.py index 5f1a0693..d4a250ea 100644 --- a/openeo_driver/processes.py +++ b/openeo_driver/processes.py @@ -426,13 +426,19 @@ def get_subset(self, names: List[str], aliases: Optional[Dict[str, str]] = None) kwargs[key] = self[alias] return kwargs - def get_enum(self, name: str, options: Collection[ArgumentValue]) -> ArgumentValue: + def get_enum( + self, name: str, options: Collection[ArgumentValue], default: Optional[ArgumentValue] = None + ) -> ArgumentValue: """ Get argument by name and check if it belongs to given set of (enum) values. Originally: `extract_arg_enum` """ - value = self.get_required(name=name) + # TODO: use an "unset" sentinel value instead of None for default? + if default is None: + value = self.get_required(name=name) + else: + value = self.get_optional(name=name, default=default) if value not in options: raise ProcessParameterInvalidException( parameter=name, diff --git a/tests/test_processes.py b/tests/test_processes.py index 663954fd..632f80ba 100644 --- a/tests/test_processes.py +++ b/tests/test_processes.py @@ -615,6 +615,15 @@ def test_get_enum(self): ): _ = args.get_enum("color", options=["R", "G", "B"]) + def test_get_enum_optional(self): + args = ProcessArgs({"size": 3, "color": "red"}, process_id="wibble") + assert args.get_enum("color", options=["red", "green", "blue"], default="green") == "red" + assert args.get_enum("colour", options=["red", "green", "blue"], default="green") == "green" + + assert args.get_enum("size", options=[0, 1, 2, 3], default=0) == 3 + assert args.get_enum("dim", options=[0, 1, 2, 3], default=0) == 0 + assert args.get_enum("dim", options=[0, 1, 2, 3], default=2) == 2 + def test_validator_generic(self): args = ProcessArgs({"size": 11}, process_id="wibble")