diff --git a/b2safe.config b/b2safe.config index 3d21e08..0b4c032 100644 --- a/b2safe.config +++ b/b2safe.config @@ -1,16 +1,24 @@ #directory of the iRODS installation +#for irods3 it is, typically, /home/user/iRODS +#for irods4 it is, typically, /var/lib/irods/iRODS IRODS_DIR: /var/lib/irods/iRODS #directory of the iRODS configuration files +#for irods3 it is, typically, /home/user/iRODS/server/config/reConfigs +#for irods4 it is, typically, /etc/irods IRODS_CONF_DIR: /etc/irods #directory of the iRODS server configuration file (server.config) +#for irods3 it is, typically, /home/user/iRODS/server/config +#for irods4 it is, typically, /etc/irods IRODS_SERVER_CONF_DIR: /etc/irods #directory of the source package downloaded via git SOURCE_DIR: /var/lib/irods/B2SAFE-core #directory of iRODS module configuration files +#for irods3 it is, typically, /home/user/b2safe-core +#for irods4 it is, typically, /etc/irods/b2safe-core CONF_DIR: /etc/irods/b2safe-core #default iRODS resource @@ -20,6 +28,8 @@ DEFAULT_RESOURCE: demoResc CRED_STORE_TYPE: os #path to the credentials file for EPIC +#for irods3 it is, typically, /home/user/b2safe-core/credentials +#for irods4 it is, typically, /etc/irods/b2safe-core/credentials CRED_FILE_PATH: /etc/irods/b2safe-core/credentials #iRODS server id in the form ://: @@ -42,6 +52,8 @@ USERS: * LOG_LEVEL: DEBUG #directory of the B2SAFE log files +#for irods3 it is, typically, /home/user/b2safe-core/log +#for irods4 it is, typically, /etc/irods/b2safe-core/log LOG_DIR: /etc/irods/b2safe-core/log #shared iRODS collection to store command files diff --git a/b2safe_install.py b/b2safe_install.py index 2c6ded2..5a39276 100755 --- a/b2safe_install.py +++ b/b2safe_install.py @@ -206,7 +206,14 @@ def check(strr, fname): + BACK).lower() chk = inpt(inp) -stat(SOURCE_DIR) +try: + subprocess.check_call(["stat", SOURCE_DIR]) +except subprocess.CalledProcessError: + print RED + BOLD + 'Directory ' + SOURCE_DIR + ' does not exist.' \ + 'Please, check the SOURCE_DIR in 2safe.config. ' \ + 'It should point to the place where you have downloaded ' \ + 'the git repository' + BACK + exit() try: subprocess.call(["cp", "-r"] + glob.glob(os.path.join(SOURCE_DIR \ @@ -238,7 +245,7 @@ def check(strr, fname): symlink(SOURCE_DIR + "/rulebase/local.re", \ IRODS_CONF_DIR + "/euloc.re", \ IRODS_CONF_DIR + "/euloc.re") - + print '3. edit /server/config/server.config and append '\ ',eudat,eurepl,eupids,eucerr,euloc'\ ',to reRuleSet (make sure to include the comma and no spaces)' @@ -267,7 +274,20 @@ def check(strr, fname): line1 = line.replace("replication","eurepl") line1 = line1.replace("pid-service","eupids") line1 = line1.replace("catchError","eucerr") + line1 = line1.replace(",eudat-authZ-filters","") + line1 = line1.replace("eudat-authZ-filters,","") + line1 = line1.replace(",authZ","") + line1 = line1.replace("authZ,","") line1 = line1.replace("local","euloc") + line1 = line1.replace(",integritycheck","") + line1 = line1.replace("integritycheck,","") + destination.write(line1) + elif (line.find('euaf') > -1) or \ +(line.find('euint') > -1): + line1 = line.replace(",euaf","") + line1 = line1.replace("euaf,","") + line1 = line1.replace(",euint","") + line1 = line1.replace("euint,","") destination.write(line1) else: destination.write(line) @@ -521,7 +541,7 @@ def check(strr, fname): else: subprocess.call(["mkdir", LOG_DIR]) else: - print RED + BOLD + 'Directory ' + LOG_DIR + ' exists. \n' + print RED + BOLD + 'Directory ' + LOG_DIR + ' exists. \n' + BACK filename = CONF_DIR + "/log.manager.conf" try: diff --git a/install.txt b/install.txt index 0674aaf..f87fa7d 100644 --- a/install.txt +++ b/install.txt @@ -21,34 +21,27 @@ Enable the module Automatically: -1.1. enable the module in "/modules/B2SAFE/info.txt" - If a previous version of the module is present, disable it before to install the new one, in "/ - modules/OLD_MODULE/info.txt" (if the directory name is the same, change it) - -1.2. rerun the irodssetup script - -Manually: - -1.1. /scripts/configure --enable-B2SAFE - If a previous version of the module is present, disable it before to install the new one (if the directory - name is the same, change it): - /scripts/configure --disable-OLD_MODULE --enable-B2SAFE -1.2. make clean -1.3. make -1.4 /irodsctl restart +0. download the git repository + from https://github.com/EUDAT-B2SAFE/B2SAFE-core/releases +1. copy source conf dir (/whre_you_have_downloaded/B2SAFE-core/conf) + to the future config directory of the b2safe module + (for irods4: /etc/irods/b2safe-core; for irods3: /home/user/b2safe-core) 2. create a symbolic link to the eudat rulebase - cd - ln -s /modules/B2SAFE/rulebase/*.re ./server/config/reConfigs/*.re + for irods3: ln -s /whre_you_have_downloaded/B2SAFE-core/rulebase/*.re /server/config/reConfigs/*.re + for irods4: ln -s /whre_you_have_downloaded/B2SAFE-core/rulebase/*.re /etc/irods/*.re + replacing the jolly character "*" with each file name available in the directory "rulebase". -3. edit /server/config/server.config and append to "reRuleSet" the list of ruleset file names written +3. edit /server/config/server.config (irods3) or /etc/irods/server.config (irods4) + and append to "reRuleSet" the list of ruleset file names written in a single line and without the suffix".re", in this way ",filename1,filename2,filename3 ..." (make sure to include the comma and no spaces) 4. configure iRODS hooks. - edit the /server/config/reConfigs/core.re file and add the following two acPostProcForPutHooks: + edit the /server/config/reConfigs/core.re (irods3) or /etc/irods/core.re (irods4) file + and add the following two acPostProcForPutHooks: acPostProcForPut { ON($objPath like "\*.replicate") { @@ -61,30 +54,40 @@ Manually: } } -5. properly configure the default resource in /server/config/reConfigs/core.re +5. properly configure the default resource in /server/config/reConfigs/core.re (irods3) + or /etc/irods/core.re (irods4) 6.1 install the python scripts - cd - ln -s /modules/B2SAFE/cmd/* ./server/bin/cmd/ + for irods3: is usually /home/user/iRODS + for irods4: is usually /var/lib/irods/iRODS + ln -s /whre_you_have_downloaded/B2SAFE-core/cmd/* /server/bin/cmd/ check permissions on the scripts and make sure they are executable by the irods user e.g.chmod u+x cmd/* -6.2 update the "getEpicApiParameters" rule in "./server/config/reConfigs/local.re" - - Configure the credential storage type: "os": stored on the local filesystem or "irods": stored on de irods namespace. +6.2 update the "getEpicApiParameters" rule in "./server/config/reConfigs/euloc.re" (irods3) + or /etc/irods/euloc.re (irods4) + - Configure the credential storage type: "os": stored on the local filesystem + or "irods": stored on de irods namespace. - Set the path to the credentials file - - set the correct serverID to include the fully qualified hostname. For instance: "irods://node.domain.com:1247" - - Set the proper values in the credentials file (see ./cmd/credentials_example for an example) -6.3 update the "getAuthZParameters" rule in "./server/config/reConfigs/local.re" - - Set the proper values in modules/B2SAFE/cmd/authz.map.json -6.4 update the "getLogParameters" rule in "./server/config/reConfigs/local.re" - - Set the proper values in modules/B2SAFE/cmd/log.manager.conf + - set the correct serverID to include the fully qualified hostname. + For instance: "irods://node.domain.com:1247" + - Set the proper values in the credentials file + (see /whre_you_have_downloaded/B2SAFE-core/cmd/credentials_example for an example) + +6.3 update the "getAuthZParameters" rule in "./server/config/reConfigs/euloc.re" (irods3) + or /etc/irods/euloc.re (irods4) + - Set the proper values in /server/bin/cmd/authz.map.json + +6.4 update the "getLogParameters" rule in "./server/config/reConfigs/euloc.re" (irods3) + or /etc/irods/euloc.re (irods4) + - Set the proper values in /server/bin/cmd/log.manager.conf -7. create a shared space in all zones as configured in the eudat.re rulebase getSharedCollection function. - - defaults to "/replicate" - - make sure all users involved in the replication can write in this collection. +7. create a shared space in all zones (local and remote), for example: "/replicate". + Grant all users involved in the replication write access to this collection. 8.0.1 change "#!/usr/bin/env python" in the python scripts in modules/B2SAFE/cmd/ to your python installation -8.0.2 install httplib2, simplejson, lxml, defusedxml, and pylint: + +8.0.2 install httplib2, simplejson, lxml, defusedxml, queuelib, and pylint: httplib2 download from http://code.google.com/p/httplib2 python setup.py install @@ -97,15 +100,25 @@ Manually: pip install defusedxml apt-get install pylint yum install pylint + + queuelib + download from https://pypi.python.org/pypi/queuelib + python setup.py install -8.1 test the epic api interaction by running the "./cmd/epicclient.py test" script manually and with "iexecmd epicclient.py" -8.2 test the replication by changing and triggering "replicate.r" rule in /modules/B2SAFE/rules +8.1 test the epic api interaction by running + "./cmd/epicclient.py test" script manually + and with "iexecmd epicclient.py" + +8.2 test the module by changing and triggering rules in /modules/B2SAFE/rules + For example, eudatRepl.r to replicate a data set. See the hints inside the rule. DATACENTER only: -it is necessary to configure the authz.map.json file in order to allow user to execute rules, which calls external scripts (see documentation). +it is necessary to configure the /server/bin/cmd/authz.map.json file +in order to allow user to execute rules, which calls external scripts (see documentation). -it is necessary to configure the log.manager.conf file in order to enable the logging system (see documentation). +it is necessary to configure the /server/bin/cmd/log.manager.conf file +in order to enable the logging system (see documentation). *** Command files *** diff --git a/packaging/create_deb_package.sh b/packaging/create_deb_package.sh index 3a9033f..2a20392 100755 --- a/packaging/create_deb_package.sh +++ b/packaging/create_deb_package.sh @@ -11,9 +11,11 @@ RPM_BUILD_ROOT="${HOME}/debbuild/" RPM_SOURCE_DIR=$B2SAFEHOME PRODUCT="irods-eudat-b2safe" IRODS_PACKAGE_DIR=`grep -i _irodsPackage ${PRODUCT}.spec | head -n 1 | awk '{print $3}'` -# retrieve parameters from spec file. So we only have to update the spec file. -VERSION=`grep -i "^Version:" ${PRODUCT}.spec | awk '{print $2}'` -RELEASE=`grep -i "^Release:" ${PRODUCT}.spec | awk '{print $2}'` +# retrieve parameters from local.re in tree +MAJOR_VERS=`grep "^\s*\*major_version" $B2SAFEHOME/rulebase/local.re | awk -F\" '{print $2}'` +MINOR_VERS=`grep "^\s*\*minor_version" $B2SAFEHOME/rulebase/local.re | awk -F\" '{print $2}'` +VERSION="${MAJOR_VERS}.${MINOR_VERS}" +RELEASE=`grep "^\s*\*sub_version" $B2SAFEHOME/rulebase/local.re | awk -F\" '{print $2}'` PACKAGE="${PRODUCT}_${VERSION}-${RELEASE}" if [ "$USERNAME" = "root" ] diff --git a/packaging/create_rpm_package.sh b/packaging/create_rpm_package.sh index 0040dcd..0c83274 100755 --- a/packaging/create_rpm_package.sh +++ b/packaging/create_rpm_package.sh @@ -30,7 +30,16 @@ else echo '%_topdir %(echo $HOME)/rpmbuild' > ~/.rpmmacros fi +# find directory where we are executing: +ABSOLUTE_PATH=$(cd `dirname "${BASH_SOURCE[0]}"` && pwd) + +#extract major_version, minor_version and subversion from local.re in tree +MAJOR_VERS=`grep "^\s*\*major_version" $ABSOLUTE_PATH/../rulebase/local.re | awk -F\" '{print $2}'` +MINOR_VERS=`grep "^\s*\*minor_version" $ABSOLUTE_PATH/../rulebase/local.re | awk -F\" '{print $2}'` +SUB_VERS=`grep "^\s*\*sub_version" $ABSOLUTE_PATH/../rulebase/local.re | awk -F\" '{print $2}'` +VERSION="${MAJOR_VERS}.${MINOR_VERS}" + # build rpm -rpmbuild -ba irods-eudat-b2safe.spec +rpmbuild -ba --define "_version $VERSION" --define "_release $SUB_VERS" irods-eudat-b2safe.spec # done.. diff --git a/packaging/install.sh b/packaging/install.sh index 5dad03c..153997f 100755 --- a/packaging/install.sh +++ b/packaging/install.sh @@ -42,6 +42,7 @@ SHARED_SPACE= # end set default parameters for installation # DATE_TODAY=`date +%Y%m%d` +JSON_CONFIG="false" # end default parameters for setup @@ -67,6 +68,11 @@ create_links() { COUNT=0 EUDAT_SERVER_CONFIG=$IRODS_CONF_DIR/server.config + if [ "$JSON_CONFIG" == "true" ] + then + EUDAT_SERVER_CONFIG=$IRODS_CONF_DIR/server_config.json + EUDAT_SERVER_CONFIG_JSON_STRING=`awk 1 ORS=' ' ${EUDAT_SERVER_CONFIG} | sed 's/ //g'` + fi #delete old symbolic links for file in $IRODS_CONF_DIR/eudat*.re @@ -81,7 +87,12 @@ create_links() { for file in $B2SAFE_PACKAGE_DIR/rulebase/*.re do LINK=eudat${COUNT} - grep "^reRuleSet.*$LINK.*" $EUDAT_SERVER_CONFIG > /dev/null + if [ "$JSON_CONFIG" == "true" ] + then + echo $EUDAT_SERVER_CONFIG_JSON_STRING | grep -e "re_rulebase_set.*$LINK.*" > /dev/null + else + cat $EUDAT_SERVER_CONFIG | grep -e "^reRuleSet.*$LINK.*" > /dev/null + fi if [ $? -ne 0 ] then EUDAT_RULEFILES="$EUDAT_RULEFILES,$LINK" @@ -141,6 +152,52 @@ update_server_config() { return $STATUS } +update_server_config_json() { + + EUDAT_SERVER_CONFIG=$IRODS_CONF_DIR/server_config.json + + if [ -n "$EUDAT_RULEFILES" ] + then + if [ ! -e ${EUDAT_SERVER_CONFIG}.org.${DATE_TODAY} ] + then + cp ${EUDAT_SERVER_CONFIG} ${EUDAT_SERVER_CONFIG}.org.${DATE_TODAY} + fi + # put json file in a single string so it easier to process. + EUDAT_SERVER_CONFIG_JSON_STRING=`awk 1 ORS=' ' ${EUDAT_SERVER_CONFIG} | sed 's/ //g'` + JSON_RULESET=`echo $EUDAT_SERVER_CONFIG_JSON_STRING | sed 's/.*re_rulebase_set":/re_rulebase_set":/' | sed 's/}].*$/}]/'` + # append rules.. + JSON_RULESET_WORK=`echo $JSON_RULESET | sed 's/]//'` + EUDAT_RULEFILES=`echo $EUDAT_RULEFILES | sed 's/,/ /g'` + for file in $EUDAT_RULEFILES + do + JSON_RULESET_WORK+=",{\"filename\":\"$file\"}" + done + JSON_RULESET_WORK+="]" + # substitute the new string for the old string + perl -pi -e "undef $/; s/re_rulebase_set[^\]]*\]/$JSON_RULESET_WORK/" $EUDAT_SERVER_CONFIG + if [ $? -eq 0 ] + then + echo "The file $EUDAT_SERVER_CONFIG has been updated!" + else + echo "ERROR: updating $EUDAT_SERVER_CONFIG failed!" + STATUS=1 + fi + fi + + echo "*********************************************************************" + echo "" + echo "Please check the file: ${EUDAT_SERVER_CONFIG} by hand " + echo " grep -A5 re_rulebase_set ${EUDAT_SERVER_CONFIG} " + echo "" + echo "It should only have the following eudat{n}.re files mentioned: " + echo " `cd $IRODS_CONF_DIR ; ls -C eudat*.re` " + echo "" + echo "*********************************************************************" + + + return $STATUS +} + configure_irods_hooks() { IRODS_COREFILE=$IRODS_CONF_DIR/core.re @@ -344,6 +401,59 @@ update_get_auth_parameters() { return $STATUS } +update_authz_map_json() { + AUTH_MAP_PATH=$B2SAFE_PACKAGE_DIR/conf/authz.map.json + TMP_FILE=/tmp/authz.map.json + if [ ! -e ${AUTH_MAP_PATH}.org.${DATE_TODAY} ] + then + cp $AUTH_MAP_PATH ${AUTH_MAP_PATH}.org.${DATE_TODAY} + fi + + # constrict a list of users to fill in the userlist + let count=0 + for user in $USERS + do + if [ "$count" -lt "1" ] + then + userlist=$user + else + userlist+="\",\"$user" + fi + let count=$count+1 + done + + cat > $TMP_FILE << EOF +{ +"assertion 1": + { "subject": + [ "$userlist" ], + "action": + [ "read" ], + "target": + [ "${IRODS_DIR}/server/bin/cmd/*","${B2SAFE_PACKAGE_DIR}/conf/*" ] + } +} + +EOF + if [ $? -eq 0 ] + then + mv $TMP_FILE $AUTH_MAP_PATH + if [ ! $? -eq 0 ] + then + echo "ERROR: moving $AUTH_MAP_PATH failed!" + STATUS=1 + fi + else + echo "ERROR: creating $TMP_FILE failed!" + STATUS=1 + fi + + # set access mode to file + chmod 600 $AUTH_MAP_PATH + + return $STATUS +} + update_get_log_parameters() { LOG_MANAGER_CONF=$B2SAFE_PACKAGE_DIR/conf/log.manager.conf @@ -404,12 +514,24 @@ update_log_manager_conf() { # main program ######################## + # # read parameter file # echo "read_parameters" read_parameters +# +# check for json or normal config files +# irods 4.1 and higher use json config files +# +echo "check iRODS config files" +if [ -e "${IRODS_DIR}/../VERSION.json" ] +then + JSON_CONFIG="true" + echo "We have iRODS 4.1 or higher. The config files are in json format" +fi + # # create symbolic links to the eudat rulebase # @@ -420,7 +542,7 @@ then fi # -# edit /etc/irods/server.config +# edit /etc/irods/server.config or /etc/irods/server_config.json # append eudat specific rules to to reRuleSet. # we use links in the type of eudatxy.re. # (make sure to include the comma and no spaces) @@ -428,7 +550,12 @@ fi if [ $? -eq 0 ] then echo "update_server_config" - update_server_config + if [ "$JSON_CONFIG" == "false" ] + then + update_server_config + else + update_server_config_json + fi fi # diff --git a/packaging/irods-eudat-b2safe.spec b/packaging/irods-eudat-b2safe.spec index c7cbc58..e123e61 100644 --- a/packaging/irods-eudat-b2safe.spec +++ b/packaging/irods-eudat-b2safe.spec @@ -1,6 +1,6 @@ Name: irods-eudat-b2safe -Version: 3.0 -Release: 0 +Version: %{_version} +Release: %{_release} #Release: 1%{?dist} Summary: b2safe core application for iRODS v4 @@ -147,6 +147,8 @@ cd %{_irodsPackage}/packaging EOF %changelog +* Mon Jul 07 2015 Robert Verkerk 3.0 +- assign version of package at build time with input parameters * Fri Feb 13 2015 Robert Verkerk 3.0 - Add files to b2safe package * Wed Jan 28 2015 Robert Verkerk 3.0 diff --git a/rules/eudatCHMC.r b/rules/eudatCHMC.r new file mode 100644 index 0000000..191260b --- /dev/null +++ b/rules/eudatCHMC.r @@ -0,0 +1,10 @@ +eudatCHMC{ + # Walk through the collection. For each object in the collection + # create a PID and stores its value and the object checksum + # in the iCAT if it does not exist. + # IF the PID exists, modify its checksum value + EUDATeiPIDeiChecksumMgmtColl(*sourceColl); +} +INPUT *sourceColl='/vzRZGEUDAT/home/eudat/11' +OUTPUT ruleExecOut + diff --git a/rules/eudatGetV.r b/rules/eudatGetV.r new file mode 100644 index 0000000..82cdbdb --- /dev/null +++ b/rules/eudatGetV.r @@ -0,0 +1,7 @@ +eudatGetV{ + # Provides version of the B2SAFE + getB2SAFEVersion(*version); +} +INPUT *source='1' +OUTPUT *version,ruleExecOut + diff --git a/rules/eudatIC.r b/rules/eudatIC.r new file mode 100644 index 0000000..6cdd324 --- /dev/null +++ b/rules/eudatIC.r @@ -0,0 +1,8 @@ +eudatIC{ + # Compare cheksums of data objects in the source and destination + # collection recursively + EUDATCheckIntegrityColl(*source, *destination, bool("true"), *response) +} +INPUT *source='/vzRZGEUDAT/home/eudat/11', *destination='/devRZG/home/eudat#vzRZGEUDAT/11-6' +OUTPUT ruleExecOut + diff --git a/rules/eudatPidsColl.r b/rules/eudatPidsColl.r new file mode 100644 index 0000000..f957c6e --- /dev/null +++ b/rules/eudatPidsColl.r @@ -0,0 +1,8 @@ +eudatPidsColl{ + # Create PIDs for all collections and objects in the collection recursively + # ROR is assumed to be "None" + EUDATPidsForColl(*coll_path); +} +INPUT *coll_path='/vzRZGEUDAT/home/eudat/11' +OUTPUT *newPID,ruleExecOut + diff --git a/rules/eudatRepl.r b/rules/eudatRepl.r new file mode 100644 index 0000000..85cc139 --- /dev/null +++ b/rules/eudatRepl.r @@ -0,0 +1,9 @@ +eudatRepl{ + # Data set replication + # registered data (with PID registration) (3rd argument - 1st bool("true")) + # recursive (4th argument 2nd bool("true")) + EUDATReplication(*source, *destination, bool("true"), bool("true"), *response) +} +INPUT *source='/vzRZGEUDAT/home/eudat/11', *destination='/devRZG/home/eudat#vzRZGEUDAT/11-6' +OUTPUT ruleExecOut + diff --git a/rules/eudatTD.r b/rules/eudatTD.r new file mode 100644 index 0000000..d23bfad --- /dev/null +++ b/rules/eudatTD.r @@ -0,0 +1,12 @@ +eudatTD{ + # Calculate the difference between the creation time + # and the modification time of an object. In seconds. + EUDATgetObjectTimeDiff(*filePath, *mode, *age); + + # Calculate the difference between the current time + # and the modification time of an object. In seconds. + EUDATgetObjectAge(*filePath, *age1); +} +INPUT *filePath='/vzRZGEUDAT/home/eudat/11/11.txt', *mode='1' +OUTPUT *age,*age1,ruleExecOut +