From dd02356f9d2d1c3ff168feaf191a455c3713f6b7 Mon Sep 17 00:00:00 2001 From: Claudio Cacciari Date: Tue, 11 Apr 2017 15:45:19 +0000 Subject: [PATCH] preparation for the new release --- changelog.txt | 11 +++++++ rulebase/catchError.re | 15 ++++----- rulebase/eudat.re | 69 +++++++++++++++-------------------------- rulebase/local.re | 31 +++++++++--------- rulebase/pid-service.re | 61 +++++++++--------------------------- rulebase/replication.re | 39 +++++++++-------------- 6 files changed, 86 insertions(+), 140 deletions(-) diff --git a/changelog.txt b/changelog.txt index ca17bae..7fcea57 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,14 @@ +B2SAFE package 4.0.0 [2017/04/12] +Tested with iRODS 4.1.x and 4.2.0. +Compatible with EPIC API and HSv8 API. +- added support for iRODS v4.2.0 +- added support for the EUDAT PID profile +- changed epic client to support new b2handle library +- python script code cleaning +- initial support of the metadata module (added a couple of rules) +- various bug fixing +- initial support for the ARGO messaging system + B2SAFE package 3.1.3 [2017/01/04] bug fixing release: - fixed many epicclents bugs diff --git a/rulebase/catchError.re b/rulebase/catchError.re index f1de9c5..bff2ac9 100644 --- a/rulebase/catchError.re +++ b/rulebase/catchError.re @@ -12,7 +12,7 @@ # EUDATCheckOwnershipObj(*path) # EUDATCheckOwnershipColl(*path) -# + # Check if 2 replicas have the same checksum # # Parameters: @@ -20,7 +20,7 @@ # *destination [IN] path destination of replicated data object # # Author: Long Phan, JSC -# +#------------------------------------------------------------------------------- EUDATCatchErrorChecksum(*source,*destination) { logInfo("[EUDATCatchErrorChecksum] Check if 2 replicas have the same checksum. " ++ "Source = *source, destination = *destination"); @@ -41,7 +41,6 @@ EUDATCatchErrorChecksum(*source,*destination) { } -# # Check if 2 replicas have the same size. # # Parameters: @@ -49,7 +48,7 @@ EUDATCatchErrorChecksum(*source,*destination) { # *destination [IN] path destination of replicated data object # # Author: Long Phan, JSC -# +#------------------------------------------------------------------------------- EUDATCatchErrorSize(*source,*destination) { logInfo("[EUDATCatchErrorSize] Check if 2 replicas have the same size." ++ "Source = *source, destination = *destination"); @@ -78,7 +77,6 @@ EUDATCatchErrorSize(*source,*destination) { } -# # Check if a user is or is not owner of the data object/collection # (Reference: https://www.irods.org/index.php/iRODS_Error_Codes or /iRODS/lib/core/include/rodsErrorTable.h) # @@ -88,7 +86,7 @@ EUDATCatchErrorSize(*source,*destination) { # # Author: Long Phan, JSC; # Author: Claudio Cacciari, Cineca; -#---------------------------------------------------- +#------------------------------------------------------------------------------- EUDATCatchErrorDataOwner(*path,*msg) { *msg = "ownership check passed"; @@ -138,7 +136,6 @@ EUDATCatchErrorDataOwner(*path,*msg) { } } - # Check if a user is or is not owner of the data object, # but comparing the session var "userNameClient" with the owners of the object # @@ -147,7 +144,7 @@ EUDATCatchErrorDataOwner(*path,*msg) { # *message [OUT] response message # # Author: Claudio Cacciari, Cineca; -#---------------------------------------------------- +#------------------------------------------------------------------------------- EUDATCheckOwnershipObj(*path) { msiSplitPath(*path, *collPath, *objPath); @@ -168,7 +165,7 @@ EUDATCheckOwnershipObj(*path) { # *message [OUT] response message # # Author: Claudio Cacciari, Cineca; -# ---------------------------------------------------- +#------------------------------------------------------------------------------- EUDATCheckOwnershipColl(*path) { *owner = 0; diff --git a/rulebase/eudat.re b/rulebase/eudat.re index 296b4ef..e4029fd 100644 --- a/rulebase/eudat.re +++ b/rulebase/eudat.re @@ -10,6 +10,7 @@ # EUDATAuthZ(*user, *action, *target, *response) #---- utility --- # EUDATisMetadata(*path) +# EUDATPushMetadata(*path, *queue) # EUDATMessage(*queue, *message) # EUDATLog(*message, *level) # EUDATQueue(*action, *message, *number) @@ -31,6 +32,8 @@ # EUDATfileInPath(*path,*subColl) # EUDATCreateAVU(*Key,*Value,*Path) # EUDATgetLastAVU(*Path, *Key, *Value) +# EUDATgetCollAVU(*path, *res) +# EUDATgetBulkMetadata(*path, *res) # EUDATcountMetaKeys( *Path, *Key, *Value ) # EUDATStoreJSONMetadata(*path, *pid, *ror, *checksum, *modtime) #---- repository packages --- @@ -45,7 +48,6 @@ # # ################################################################################ -# # Return a boolean value: # True, if the authorization request matches against, at least # one assertion listed in the authz.map.json file @@ -58,7 +60,7 @@ # *response [OUT] True or False depending on authorization rights # # Author: Claudio Cacciari, Cineca -# +#------------------------------------------------------------------------------- EUDATAuthZ(*user, *action, *target, *response) { getAuthZParameters(*authZMapPath); logDebug("checking authorization for *user to perform: *action *target"); @@ -99,7 +101,7 @@ EUDATAuthZ(*user, *action, *target, *response) { # *path [IN] the path of the object/collection # # Author: Claudio Cacciari, Cineca -# +#------------------------------------------------------------------------------- EUDATisMetadata(*path) { *isMeta = bool("false"); if (*path like regex ".*\\.metadata.*") { @@ -115,7 +117,7 @@ EUDATisMetadata(*path) { # *path [IN] the path of the object/collection # # Author: Claudio Cacciari, Cineca -# +#------------------------------------------------------------------------------- EUDATPushMetadata(*path, *queue) { logInfo("[EUDATPushMetadata] pushing metadata of object *path to topic *queue") @@ -141,7 +143,7 @@ EUDATPushMetadata(*path, *queue) { # *message [IN] the message to be sent # # Author: Claudio Cacciari, Cineca -# +#------------------------------------------------------------------------------- EUDATMessage(*queue, *message) { logInfo("[EUDATMessage] pushing the message to topic *queue"); @@ -157,7 +159,6 @@ EUDATMessage(*queue, *message) { } } - # It manages the writing and reading of log messages to/from external log services. # The current implementation writes the logs to specific log file. # @@ -169,7 +170,7 @@ EUDATMessage(*queue, *message) { # *level [IN] the logging level # # Author: Claudio Cacciari, Cineca -# +#------------------------------------------------------------------------------- EUDATLog(*message, *level) { getLogParameters(*logConfPath); logInfo("logging message '*message'"); @@ -181,7 +182,6 @@ EUDATLog(*message, *level) { } } - # It implements a FIFO queue for messages to/from external log services. # # Return @@ -196,7 +196,7 @@ EUDATLog(*message, *level) { # *number [IN] the number of elements to be extracted # # Author: Claudio Cacciari, Cineca -# +#------------------------------------------------------------------------------- EUDATQueue(*action, *message, *number) { getLogParameters(*logConfPath); *options = ""; @@ -222,9 +222,11 @@ EUDATQueue(*action, *message, *number) { } } -# -# Logging policies -# +################################################################################ +# # +# Logging policies # +# # +################################################################################ logDebug(*msg) { getEUDATLoggerLevel(*level); @@ -253,7 +255,6 @@ logWithLevel(*level, *msg) { on (*level == "error") { writeLine("serverLog","ERROR: *msg");} } -#----------------------------------------------- # Function: trsnsfrom string to boolean value # # Author: Claudio Cacciari (Cineca) @@ -269,11 +270,10 @@ EUDATtoBoolean(*var) { *status } -# # Function: replace epicclient function # # Author: Robert Verkerk SURFsara -# +#------------------------------------------------------------------------------- EUDATReplaceHash(*path, *out) { # replace # @@ -289,18 +289,16 @@ EUDATReplaceHash(*path, *out) { } } -# # Function: replace microservice msiGetZoneNameFromPath (eudat.c) # # Author: Long Phan, JSC -# +#------------------------------------------------------------------------------- EUDATGetZoneNameFromPath(*path, *out) { *list = split("*path","/"); *out = elem(*list,0); } -# # Gets the connection details of an iRODS Zone. # # Arguments: @@ -308,7 +306,7 @@ EUDATGetZoneNameFromPath(*path, *out) { # *conn [OUT] the connection details related to the input iRODS Zone (hostname:port) # # Author: Claudio Cacciari, Cineca -#---------------------------------------------------------- +#------------------------------------------------------------------------------- EUDATGetZoneHostFromZoneName(*zoneName, *conn) { *conn = "" @@ -329,7 +327,6 @@ EUDATGetZoneHostFromZoneName(*zoneName, *conn) { } } -# # Checks if date of the last computation of iCHECKSUM was set and set the date if not. # The date is stored as metadata attribute of name 'eudat_dpm_checksum_date:' # @@ -343,7 +340,7 @@ EUDATGetZoneHostFromZoneName(*zoneName, *conn) { # - will be assumed as time of the first computation of the iCHECKSUM # # Author: Michal Jankowski, PSNC -# +#------------------------------------------------------------------------------- EUDATiCHECKSUMdate(*coll, *name, *resc, *modTime) { *metaName = 'eudat_dpm_checksum_date:*resc'; @@ -361,8 +358,6 @@ EUDATiCHECKSUMdate(*coll, *name, *resc, *modTime) { } } - -# # The function retrieve iCHECKSUM for a given object. # # Environment variable used: @@ -374,7 +369,7 @@ EUDATiCHECKSUMdate(*coll, *name, *resc, *modTime) { # *status [REI] false if no value is found, true elsewhere # # Author: Giacomo Mariani, CINECA, Michal Jankowski PSNC -# +#------------------------------------------------------------------------------- EUDATiCHECKSUMretrieve(*path, *checksum, *modtime) { *status = bool("false"); *checksum = ""; @@ -402,7 +397,6 @@ EUDATiCHECKSUMretrieve(*path, *checksum, *modtime) { *status; } - # The function obtain iCHECKSUM for a given object creating it if necessary. # # Arguments: @@ -411,7 +405,7 @@ EUDATiCHECKSUMretrieve(*path, *checksum, *modtime) { # *modtime [OUT] modification time of the checksum # # Author: Giacomo Mariani, CINECA, Michal Jankowski PSNC - +#------------------------------------------------------------------------------- EUDATiCHECKSUMget(*path, *checksum, *modtime) { if (!EUDATiCHECKSUMretrieve(*path, *checksum, *modtime)) { #If it is a collection, do not calculate the checksum @@ -424,7 +418,6 @@ EUDATiCHECKSUMget(*path, *checksum, *modtime) { } } - # Calculate the difference between the creation time or the current time # and the modification time of an object. In seconds. # @@ -437,7 +430,7 @@ EUDATiCHECKSUMget(*path, *checksum, *modtime) { # *age = -1 if the object does not exist # # Author: Giacomo Mariani, CINECA -# +#------------------------------------------------------------------------------- EUDATgetObjectTimeDiff(*filePath, *mode, *age) { *age = -1; # Check if the file exists @@ -464,7 +457,6 @@ EUDATgetObjectTimeDiff(*filePath, *mode, *age) { } } -# # Calculate the difference between the current time and the modification time of an object. # In seconds. # @@ -474,7 +466,7 @@ EUDATgetObjectTimeDiff(*filePath, *mode, *age) { # *age = -1 if the object does not exist # # Author: Claudio Cacciari, CINECA -# +#------------------------------------------------------------------------------- EUDATgetObjectAge(*filePath, *age) { *age = -1; # Check if the file exists @@ -487,7 +479,6 @@ EUDATgetObjectAge(*filePath, *age) { *age; } -# # Rules to check if a file is in a given path. # # Arguments: @@ -496,7 +487,7 @@ EUDATgetObjectAge(*filePath, *age) { # *b [REI] False if no value is found, trou elsewhere # # Author: Hao Xu, DICE; Giacomo Mariani, CINECA -# +#------------------------------------------------------------------------------- EUDATfileInPath(*path,*subColl) { logInfo("conditional acPostProcForCopy -> EUDATfileInPath"); msiSplitPath(*path, *coll, *name); @@ -511,7 +502,6 @@ EUDATfileInPath(*path,*subColl) { *b; } -# # Create AVU with INPUT *Key, *Value for DataObj *Path # # Parameters: @@ -521,7 +511,7 @@ EUDATfileInPath(*path,*subColl) { # # Author: Long Phan, JSC # Modified: Elena Erastova, RZG, 27.08.2015 -# +#------------------------------------------------------------------------------- EUDATCreateAVU(*Key, *Value, *Path) { logDebug("[EUDATCreateAVU] Adding AVU: *Key = *Value to metadata of *Path"); msiAddKeyVal(*Keyval, *Key, *Value); @@ -529,7 +519,6 @@ EUDATCreateAVU(*Key, *Value, *Path) { msiSetKeyValuePairsToObj(*Keyval, *Path, *objType); } -#----------------------------------------------------------------------------- # get the single value of a specific metadata in ICAT # # Parameters: @@ -548,7 +537,6 @@ EUDATgetLastAVU(*Path, *Key, *Value) } } -#----------------------------------------------------------------------------- # Get all the AVUs of a collection # # Parameters: @@ -569,7 +557,6 @@ EUDATgetCollAVU(*path, *res) logDebug("[EUDATgetCollAVU] AVUs: *res"); } -#----------------------------------------------------------------------------- # Get all the AVUs of the objects under a collection # # Parameters: @@ -598,7 +585,6 @@ EUDATgetBulkMetadata(*path, *res) logDebug("[EUDATgetBulkMetadata] metadata: *res"); } -#----------------------------------------------------------------------------- # count metadata in ICAT # # Parameters: @@ -630,7 +616,7 @@ EUDATcountMetaKeys( *Path, *Key, *Value ) # *modtime [IN] The modification time of the checksum # # Author: Claudio Cacciari, CINECA - +#------------------------------------------------------------------------------- EUDATStoreJSONMetadata(*path, *pid, *ror, *checksum, *modtime) { getMetaParameters(*metaConfPath,*enabled); @@ -658,14 +644,12 @@ EUDATStoreJSONMetadata(*path, *pid, *ror, *checksum, *modtime) { } } - ################################################################################ # # # Repository Packages # # # ################################################################################ -#----------------------------------------------------------------------------- # Check if the ADMIN_Status value is set to Ready ToArchive and then kicks off ingestion # # Parameters: @@ -684,7 +668,6 @@ EUDATrp_checkMeta(*source,*AName,*AValue) } } -#----------------------------------------------------------------------------- # Manage the ingestion in B2SAFE # Check the checksum # Create PID @@ -733,7 +716,6 @@ EUDATrp_ingestObject( *source ) } } -#----------------------------------------------------------------------------- # Process executed when a transfer has been initiated # (this process is triggered by the iputPreProc hook) # @@ -749,7 +731,6 @@ EUDATrp_transferInitiated( *source ) EUDATCreateAVU("INFO_TimeOfStart", *TimeNow, *source); } -#----------------------------------------------------------------------------- # Process executed after a transfer is finished # (this process is triggered by the iputPostProc hook) # diff --git a/rulebase/local.re b/rulebase/local.re index a64f1ed..ca8f048 100644 --- a/rulebase/local.re +++ b/rulebase/local.re @@ -9,11 +9,11 @@ # Arguments: # *euLogLevel [OUT] the debug level for the EUDAT specific rules [0 | 1 | 2] # 0:ERROR, 1:INFO, 2:DEBUG +#------------------------------------------------------------------------------- getEUDATLoggerLevel(*euLogLevel) { *euLogLevel=2 } -# # Provides parameters for the connection with the EPIC service # # Arguments: @@ -25,7 +25,7 @@ getEUDATLoggerLevel(*euLogLevel) { # *epicDebug [OUT] the debug level for the EPIC client scripts # # Author: Willem Elbers (MPI-PL) -# +#------------------------------------------------------------------------------- getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug) { *credStoreType="os"; *credStorePath="/srv/irods/current/modules/B2SAFE/cmd/credentials_test"; @@ -39,7 +39,6 @@ getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicD } } -# ----------------------------------------------------------------------------- # Parse the credentials to connect to an EPIC server. A file called # "credentials" MUST contain all the connection details in the home folder of # the user running this rule. @@ -50,8 +49,7 @@ getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicD # *password [OUT] password # # Author: Javier Quinteros, RZG -# ----------------------------------------------------------------------------- -# +#------------------------------------------------------------------------------- parseCredentials (*baseuri, *username, *prefix, *password) { *baseuri = 'https://epic3.storage.surfsara.nl/v2_test/handles/' @@ -72,7 +70,7 @@ parseCredentials (*baseuri, *username, *prefix, *password) { # containing the authorization assertions. # # Author: Claudio Cacciari (Cineca) -# +#------------------------------------------------------------------------------- getAuthZParameters(*authZMapPath) { *authZMapPath="/srv/irods/current/modules/B2SAFE/cmd/authz.map.json"; } @@ -83,7 +81,7 @@ getAuthZParameters(*authZMapPath) { # *logConfPath [OUT] the file path to the logging configuration. # # Author: Claudio Cacciari (Cineca) -# +#------------------------------------------------------------------------------- getLogParameters(*logConfPath) { *logConfPath="/srv/irods/current/modules/B2SAFE/cmd/log.manager.conf"; } @@ -97,7 +95,7 @@ getLogParameters(*logConfPath) { # json metadata writing # # Author: Claudio Cacciari (Cineca) - +# ------------------------------------------------------------------------------- getMetaParameters(*metaConfPath, *enabled) { *metaConfPath="/srv/irods/current/modules/B2SAFE/cmd/metadataManager.conf"; *enabled=bool("false"); @@ -111,13 +109,13 @@ getMetaParameters(*metaConfPath, *enabled) { # the usage of the messaging system # # Author: Claudio Cacciari (Cineca) -# +#------------------------------------------------------------------------------- getMessageParameters(*msgConfPath, *enabled) { *msgLogPath="/opt/eudat/b2safe/conf/msgManager.conf"; *enabled=bool("false"); } -#Provides parameters for some B2SAFE configurations. +# Provides parameters for some B2SAFE configurations. # The plugin msifree_microservice_out is a bug fixing to avoid memory leak # in case of collection replication involving thousands of files and the # parameter msiFreeEnabled enables it. @@ -130,7 +128,7 @@ getMessageParameters(*msgConfPath, *enabled) { # defined in the file retrieved by getAuthZParameters. # # Author: Claudio Cacciari (Cineca) -# +#------------------------------------------------------------------------------- getConfParameters(*msiFreeEnabled, *msiCurlEnabled, *authzEnabled) { *msiFreeEnabled=bool("true"); *authzEnabled=bool("true"); @@ -143,15 +141,14 @@ getConfParameters(*msiFreeEnabled, *msiCurlEnabled, *authzEnabled) { # *version [OUT] the B2SAFE version. # # Author: Claudio Cacciari (Cineca) -# +#------------------------------------------------------------------------------- getB2SAFEVersion(*version) { - *major_version = "3"; - *minor_version = "9"; - *sub_version = "1"; + *major_version = "4"; + *minor_version = "0"; + *sub_version = "0"; *version = *major_version ++ "." ++ *minor_version ++ "-" ++ *sub_version; } -# # This function is used to set up some parameters for the site in case you are # going to use the EUDAT repository packages procedure to ingest data. # @@ -160,7 +157,7 @@ getB2SAFEVersion(*version) { # *archiveOwner [OUT] This is the iRods user owning the archive # # Author: S Coutin (CINES) -# +#------------------------------------------------------------------------------- rp_getRpIngestParameters(*protectArchive, *archiveOwner) { *protectArchive = false; *archiveOwner = "rodsA"; diff --git a/rulebase/pid-service.re b/rulebase/pid-service.re index 89ee811..e8123c3 100644 --- a/rulebase/pid-service.re +++ b/rulebase/pid-service.re @@ -18,11 +18,10 @@ # EUDATSearchPIDchecksum(*path, *existing_pid) # EUDATUpdatePIDWithNewChild(*parentPID, *childPID) # EUDATGeteValPid(*pid, *key) -# EUDATGeteRorPid(*pid, *ror) # EUDATeiChecksumMgmt(*path, *PID) # EUDATiPIDcreate(*path, *PID) # EUDATiFieldVALUEretrieve(*path, *FNAME, *FVALUE) -# EUDATePIDcreate(*path, *extraType, *PID, *ifchecksum) +# EUDATePIDcreate(*path, *extraType, *PID) # EUDATePIDsearch(*field, *value, *PID) # EUDATeCHECKSUMupdate(*PID, *path) # EUDATeURLupdate(*PID, *newURL) @@ -31,12 +30,12 @@ # EUDATeiChecksumMgmtColl(*sourceColl) # EUDATiRORupdate(*source, *pid) # EUDATeRORupdate(*pid,*newRor) +# EUDATeFIOupdate(*pid, *newFio) # EUDATPidsForColl(*collPath) # EUDATePIDcreateCurl(*path, *extraType, *PID, *ifchecksum) # EUDATSearchPIDCurl(*path, *existing_pid) -# # Generate a new PID for a digital object. # Fields stored in the PID record: URL, ROR and CHECKSUM # adds a ROR field if (*ror != "None") @@ -51,7 +50,7 @@ # # Author: Willem Elbers, MPI-TLA # Edited by Elena Erastova, RZG; Long Phan, JSC; Robert Verkerk, SURFsara, Javier Quinteros, GFZ -# +#------------------------------------------------------------------------------- EUDATCreatePID(*parent_pid, *path, *ror, *fio, *fixed, *newPID) { logInfo("[EUDATCreatePID] create pid for *path"); @@ -169,7 +168,6 @@ EUDATCreatePID(*parent_pid, *path, *ror, *fio, *fixed, *newPID) { } } -# ----------------------------------------------------------------------------- # Searching for a PID using URL replacing "#", "%" and "&" with "*" # Parameters: # *path [IN] the path of the replica @@ -186,14 +184,13 @@ EUDATSearchPID(*path, *existing_pid) { *status; } -# # Searching fo a PID using CHECKSUM # Parameters: # *path [IN] the path of the replica # *existing_pid [OUT] existing PID # # Author: Elena Erastova, RZG -# +#------------------------------------------------------------------------------- EUDATSearchPIDchecksum(*path, *existing_pid) { logInfo("search pid for *path"); getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug); @@ -224,7 +221,6 @@ EUDATSearchPIDchecksum(*path, *existing_pid) { } } -# # Update a PID record with a new child. # # Parameters: @@ -233,7 +229,7 @@ EUDATSearchPIDchecksum(*path, *existing_pid) { # # Author: Willem Elbers, MPI-TLA # Modified by: Claudio Cacciari, CINECA -# +#------------------------------------------------------------------------------- EUDATUpdatePIDWithNewChild(*parentPID, *childPID) { *replicaNew = "None" logInfo("[EUDATUpdatePIDWithNewChild] update parent pid (*parentPID) with new child (*childPID)"); @@ -258,7 +254,6 @@ EUDATUpdatePIDWithNewChild(*parentPID, *childPID) { *replicaNew } -#------------------------------------------------------------------------------- # get the KEY entry for a PID # # Parameters: @@ -283,17 +278,15 @@ EUDATGeteValPid(*pid, *key) { *val } -# # This function creates a PID and stores its value and the checksum in the iCAT if it does not exist. # Otherwhise the function modifies the PID. # # Arguments: # *path [IN] Path of the source file # *PID [OUT] PID of the source file -# *iCATCache [IN] Specify whether you want to use the iCAT (bool("true")) or not # # Author: Giacomo Mariani, CINECA; Claudio Cacciari CINECA; -# +#------------------------------------------------------------------------------- EUDATeiChecksumMgmt(*path, *PID) { *PID = "empty"; @@ -310,7 +303,6 @@ EUDATeiChecksumMgmt(*path, *PID) { } } -# ----------------------------------------------------------------------------- # The function write iPID given ePID. # # Parameters: @@ -342,7 +334,6 @@ EUDATiPIDcreate(*path, *PID) { *status0; } -# # The function retrieves the value of the required field from iCAT. # # Arguments: @@ -352,7 +343,7 @@ EUDATiPIDcreate(*path, *PID) { # *status0 [REI] false if no value is found, true elsewhere # # Author: Giacomo Mariani, CINECA -# +#------------------------------------------------------------------------------- EUDATiFieldVALUEretrieve(*path, *FNAME, *FVALUE) { logInfo("EUDATiFieldVALUEretrieve -> looking for *FNAME of *path"); *status0 = bool("false"); @@ -377,7 +368,6 @@ EUDATiFieldVALUEretrieve(*path, *FNAME, *FVALUE) { *status0; } -# # The function create ePID. # # Environment variable used: @@ -389,7 +379,7 @@ EUDATiFieldVALUEretrieve(*path, *FNAME, *FVALUE) { # # Author: Giacomo Mariani, CINECA # Edited by: Robert Verkerk, SURFsara -# +#------------------------------------------------------------------------------- EUDATePIDcreate(*path, *extraType, *PID) { getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug) ; @@ -431,7 +421,6 @@ EUDATePIDcreate(*path, *extraType, *PID) { logInfo("[EUDATePIDcreate] Created handle is: *PID"); } -# # The function retrieve ePID searching for a field between URL, CHECKSUM. # # Environment variable used: @@ -444,7 +433,7 @@ EUDATePIDcreate(*path, *extraType, *PID) { # *status0 [REI] false if no value is found, true elsewhere # # Author: Giacomo Mariani, CINECA -# +#------------------------------------------------------------------------------- EUDATePIDsearch(*field, *value, *PID) { logInfo("[EUDATePIDsearch] search the PID with *field = *value"); getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug); @@ -487,7 +476,7 @@ EUDATePIDsearch(*field, *value, *PID) { # *path [IN] Object path # # Author: Giacomo Mariani, CINECA - +# ------------------------------------------------------------------------------- EUDATeCHECKSUMupdate(*PID, *path) { getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug); logDebug("[EUDATeCHECKSUMupdate] modify checksum related to PID *PID"); @@ -512,7 +501,7 @@ EUDATeCHECKSUMupdate(*PID, *path) { # *newURL [IN] The new URL to be associated to the PID of $objPath # # Author: Giacomo Mariani, CINECA -# +#------------------------------------------------------------------------------- EUDATeURLupdate(*PID, *newURL) { getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug); EUDATeURLsearch(*PID, *oldURL); @@ -525,15 +514,8 @@ EUDATeURLupdate(*PID, *newURL) { msifree_microservice_out(*outEUU); } logInfo("EUDATeURLupdate -> modify handle response = *response"); - msiExecCmd("epicclient.py","*credStoreType *credStorePath modify *PID 10320/LOC \"*oldURL\" \"*newURL\"", - "null", "null", "null", *outEUUL); - getConfParameters(*msiFreeEnabled, *msiCurlEnabled, *authzEnabled); - if (*msiFreeEnabled) { - msifree_microservice_out(*outEUUL); - } } -# # This function search the URL field of the PID # # Arguments: @@ -541,7 +523,7 @@ EUDATeURLupdate(*PID, *newURL) { # *newURL [IN] The new URL to be associated to the PID # # Author: Giacomo Mariani, CINECA -# +#------------------------------------------------------------------------------- EUDATeURLsearch(*PID, *URL) { getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug); logInfo("EUDATeURLsearch -> search URL in PID *PID"); @@ -555,16 +537,14 @@ EUDATeURLsearch(*PID, *URL) { logInfo("EUDATeURLsearch -> response = *URL"); } - -# -# This function remove an ePID... even if its 10320/loc field is not empty! +# This function remove an ePID... even if its EUDAT/REPLICA field is not empty! # To be improved. # # Arguments: # *path [IN] The path of the object to be removed # # Author: Giacomo Mariani, CINECA -# +#------------------------------------------------------------------------------- EUDATePIDremove(*path, *force) { getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug) logInfo("[EUDATePIDremove] removing PID associated to: $userNameClient, *path"); @@ -587,8 +567,6 @@ EUDATePIDremove(*path, *force) { msifree_microservice_out(*outEPR1); } logInfo("[EUDATePIDremove] removing completed, response = *response3"); - # The PID record could be associated to a replica. - # The field 10320/LOC of the parent PID record should be updated } else if (EUDATtoBoolean(*force) == bool("true")){ logDebug("[EUDATePIDremove] Found replicas: PID *pid will be deleted"); @@ -614,7 +592,6 @@ EUDATePIDremove(*path, *force) { } } -# ----------------------------------------------------------------------------- # Walk through the collection. For each object in the collection # it creates the object checksum in the iCAT if it does not exist. # Otherwhise the function modify the PID. @@ -633,7 +610,6 @@ EUDATeiChecksumMgmtColl(*sourceColl) { } } -# ----------------------------------------------------------------------------- # Add or modify the ROR field of the PID of the object to iCAT # # Arguments: @@ -657,7 +633,6 @@ EUDATiRORupdate(*source, *pid) { } } -#----------------------------------------------------------------------------- # Add or modify the ROR field of the PID in EPIC system # # Arguments: @@ -679,7 +654,6 @@ EUDATeRORupdate(*pid, *newRor) { logInfo("[EUDATeRORupdate] modify handle response = *response"); } -#----------------------------------------------------------------------------- # Add or modify the FIO field of the PID in EPIC system # # Arguments: @@ -700,7 +674,6 @@ EUDATeFIOupdate(*pid, *newFio) { logInfo("[EUDATeFIOupdate] modify handle response = *response"); } -#----------------------------------------------------------------------------- # Create PIDs for all collections and objects in the collection recursively # ROR is assumed to be "None" # @@ -736,8 +709,6 @@ EUDATPidsForColl(*collPath, *fixed) { # The function create ePID. # -# Environment variable used: -# # Arguments: # *path [IN] The full iRODS path of the object # *extraType [IN] extra parameters @@ -747,7 +718,7 @@ EUDATPidsForColl(*collPath, *fixed) { # Author: Giacomo Mariani, CINECA # Edited by: Robert Verkerk, SURFsara # Edited by: Javier Quinteros, GFZ - +# ------------------------------------------------------------------------------- EUDATePIDcreateCurl(*path, *extraType, *PID, *ifcheksum) { getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug) ; @@ -817,7 +788,6 @@ EUDATePIDcreateCurl(*path, *extraType, *PID, *ifcheksum) { logInfo("EUDATePIDcreateCurl -> Created handle is: *PID"); } -# ----------------------------------------------------------------------------- # Searching for a PID using URL replacing "#", "%" and "&" with "*" # Parameters: # *path [IN] the path of the replica @@ -826,7 +796,6 @@ EUDATePIDcreateCurl(*path, *extraType, *PID, *ifcheksum) { # # Author: Javier Quinteros, GFZ # ----------------------------------------------------------------------------- - EUDATSearchPIDCurl(*path, *existing_pid) { logInfo("search pid for *path"); getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug); diff --git a/rulebase/replication.re b/rulebase/replication.re index 703e3f8..bb684f0 100644 --- a/rulebase/replication.re +++ b/rulebase/replication.re @@ -12,15 +12,15 @@ # EUDATUpdateLogging(*status_transfer_success, *source, *destination, *cause) # EUDATCheckIntegrity(*source,*destination,*logEnabled,*notification,*response) # EUDATReplication(*source, *destination, *registered, *recursive) -# EUDATTransferUsingFailLog(*buffer_length) -# EUDATRegDataRepl(*source, *destination) +# EUDATTransferUsingFailLog(*buffer_length, *stats) +# EUDATRegDataRepl(*source, *destination, *recursive, *response) # EUDATPIDRegistration(*source, *destination, *notification, *registration_response) # EUDATSearchAndCreatePID(*path, *pid) -# EUDATSearchAndDefineRoR(*path, *pid, *ROR) +# EUDATSearchAndDefineField(*path, *pid, *key) # EUDATCheckIntegrityColl(*sCollPath, *dCollPath, *logEnabled, *response) # EUDATCheckIntegrityDO(*source,*destination,*logEnabled,*response) -# + # Update the logging files specific for EUDAT B2SAFE # # Parameters: @@ -31,7 +31,7 @@ # # Author: Long Phan, JSC # Modified by Claudio Cacciari, Cineca -# +#------------------------------------------------------------------------------- EUDATUpdateLogging(*status_transfer_success, *source, *destination, *cause) { # Update Logging Statistical File @@ -44,7 +44,6 @@ EUDATUpdateLogging(*status_transfer_success, *source, *destination, *cause) { EUDATLog(*message, *level); } -# # Checks differences about checksum and size between two paths # # Parameters: @@ -55,10 +54,9 @@ EUDATUpdateLogging(*status_transfer_success, *source, *destination, *cause) { # *notification [IN] value [0|1]: if 1 enable the notification via messaging system # *response [OUT] the reason of the failure # -# # Author: Long Phan, JSC # Modified by Claudio Cacciari, Cineca -# +#------------------------------------------------------------------------------- EUDATCheckIntegrity(*source,*destination,*logEnabled,*notification,*response) { *status_transfer_success = bool("true"); @@ -88,7 +86,6 @@ EUDATCheckIntegrity(*source,*destination,*logEnabled,*notification,*response) { *status_transfer_success; } -# # Data set replication # # Parameters: @@ -97,10 +94,11 @@ EUDATCheckIntegrity(*source,*destination,*logEnabled,*notification,*response) { # *registered [IN] boolean value: "true" for registered data, "false" otherwise # *recursive [IN] boolean value: "true" to enable the recursive replication # of registered data, "false" otherwise. +# *response [OUT]the result of the replication # # Author: Long Phan, JSC # Modified by Claudio Cacciari, Cineca -# +#------------------------------------------------------------------------------- EUDATReplication(*source, *destination, *registered, *recursive, *response) { logInfo("[EUDATReplication] transfering *source to *destination"); @@ -148,17 +146,17 @@ EUDATReplication(*source, *destination, *registered, *recursive, *response) { *status; } -# # Transfer all data object saved in the logging system, # according to the format: cause::path_of_transfer_file::target_of_transfer_file. # # Parameters: # *buffer_length [IN] max number of failed transfers to process. # It has to be > 1. +# *stats [OUT] the number of successful transfers # # Author: Long Phan, JSC # Modified by Claudio Cacciari, Cineca; -# +#------------------------------------------------------------------------------- EUDATTransferUsingFailLog(*buffer_length, *stats) { logInfo("[EUDATTransferUsingFailLog] checking the last *buffer_length failed transfers"); @@ -206,8 +204,6 @@ EUDATTransferUsingFailLog(*buffer_length, *stats) { } - -#----------------------------------------------------------------------------- # Data object replication and PID management based on remote execution # It is assumed that iRODS zone federation is established # and the replicated file is accessible from the source @@ -324,15 +320,14 @@ EUDATRegDataRepl(*source, *destination, *recursive, *response) { *status; } -#----------------------------------------------------------------------------- # Verify that a PID exist for a given path and optionally create it # if not found. # # Parameters: -# *source [IN] source iRODS path -# *destination [IN] target iRODS path -# *notification [IN] enable messaging for async call [0|1] -# *response [OUT] a message containing the reason of the failure +# *source [IN] source iRODS path +# *destination [IN] target iRODS path +# *notification [IN] enable messaging for async call [0|1] +# *registration_response [OUT] a message containing the reason of the failure # # Author: Claudio, Cineca #----------------------------------------------------------------------------- @@ -407,7 +402,6 @@ EUDATPIDRegistration(*source, *destination, *notification, *registration_respons } } -#----------------------------------------------------------------------------- # Search PID for a given path and in case it is not found, # it creates a new PID. # @@ -429,7 +423,6 @@ EUDATSearchAndCreatePID(*path, *pid) { } } -#----------------------------------------------------------------------------- # Search key/value pair for a given path # # Parameters: @@ -458,7 +451,6 @@ EUDATSearchAndDefineField(*path, *pid, *key) { *val } -#---------------------------------------------------------------------------- # Compare cheksums of data objects in the source and destination # collection recursively # @@ -517,7 +509,6 @@ EUDATCheckIntegrityColl(*sCollPath, *dCollPath, *logEnabled, *check_response) { *totalResult; } -# # Checks differences about checksum and size between two Data Objects # and log the result to the B2SAFE logging system # @@ -529,7 +520,7 @@ EUDATCheckIntegrityColl(*sCollPath, *dCollPath, *logEnabled, *check_response) { # *response [OUT] the reason of the failure # # Author: Claudio Cacciari, Cineca -# +#------------------------------------------------------------------------------- EUDATCheckIntegrityDO(*source,*destination,*logEnabled,*response) { *status = bool("true");