From 86b4f4341b521f18530d4bb97833860a3c4028f6 Mon Sep 17 00:00:00 2001 From: Jari Voutilainen Date: Wed, 26 Feb 2025 16:19:50 +0200 Subject: [PATCH] AV-2418: Add validate packages feature to syke harvester --- .../ckanext/ytp/harvesters/syke_harvester.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/ckan/ckanext/ckanext-ytp_main/ckanext/ytp/harvesters/syke_harvester.py b/ckan/ckanext/ckanext-ytp_main/ckanext/ytp/harvesters/syke_harvester.py index 42787eda9e..8f92c7c916 100644 --- a/ckan/ckanext/ckanext-ytp_main/ckanext/ytp/harvesters/syke_harvester.py +++ b/ckan/ckanext/ckanext-ytp_main/ckanext/ytp/harvesters/syke_harvester.py @@ -9,6 +9,7 @@ from ckan.lib.munge import munge_tag from ckanext.harvest.model import HarvestObject from ckanext.harvest.harvesters.ckanharvester import CKANHarvester +import ckan.lib.plugins as lib_plugins import logging log = logging.getLogger(__name__) @@ -436,6 +437,42 @@ def get_extra(key, package_dict): package_dict = self.modify_package_dict(package_dict, harvest_object) + # validate packages if needed + validate_packages = self.config.get('validate_packages', {}) + if validate_packages: + if 'type' not in package_dict: + package_plugin = lib_plugins.lookup_package_plugin() + try: + # use first type as default if user didn't provide type + package_type = package_plugin.package_types()[0] + except (AttributeError, IndexError): + package_type = 'dataset' + # in case a 'dataset' plugin was registered w/o fallback + package_plugin = lib_plugins.lookup_package_plugin(package_type) + package_dict['type'] = package_type + else: + package_plugin = lib_plugins.lookup_package_plugin(package_dict['type']) + + errors = {} + # if package has been previously imported + try: + existing_package_dict = self._find_existing_package(package_dict) + + if 'metadata_modified' not in package_dict or \ + package_dict['metadata_modified'] > existing_package_dict.get('metadata_modified'): + schema = package_plugin.update_package_schema() + data, errors = lib_plugins.plugin_validate( + package_plugin, base_context, package_dict, schema, 'package_update') + + except NotFound: + + schema = package_plugin.create_package_schema() + data, errors = lib_plugins.plugin_validate( + package_plugin, base_context, package_dict, schema, 'package_create') + + if errors: + raise ValidationError(errors) + result = self._create_or_update_package( package_dict, harvest_object, package_dict_form='package_show')