diff --git a/scripts/conf/remote.users.sync.conf b/scripts/conf/remote.users.sync.conf new file mode 100644 index 0000000..d3ad886 --- /dev/null +++ b/scripts/conf/remote.users.sync.conf @@ -0,0 +1,28 @@ +# section containing the common options +[Common] +logfile=remote.sync.log +# possible values: INFO, DEBUG, ERROR, WARNING +loglevel=DEBUG +usersfile=test/irods.external +#dnsfile=test/irods.DNs.map + +# section for HBP_prjtome +[HBP_prjtome] + +# section for EUHIT_Repo +[EUHIT_Repo] +ckan_api_root=xxxxxxxxxxxx +admin_api_key=xxxxxxxxxxxxx +# default_groups=proj1, proj2, proj3 ... +# groups intended for all the users +default_groups=EUHIT_Read + +# section for MyProxy +[MyProxy] +irods_path_to_dns=/CINECA01/home/proirod1/validDN +gridftp_cert_dn=/C=IT/O=INFN/OU=Host/L=CINECA-SAP/CN=data.repo.cineca.it + +[EUDAT] +host=https://unity.eudat-aai.fz-juelich.de:8443/rest-admin/v1/ +username= +password= \ No newline at end of file diff --git a/scripts/remote.users.sync.py b/scripts/remote.users.sync.py new file mode 100755 index 0000000..d9fa277 --- /dev/null +++ b/scripts/remote.users.sync.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python +# -*- python -*- + +import sys +import argparse +import json +import subprocess +import logging +import logging.handlers +import urllib2 +import urllib +from pprint import pformat +import os +import ConfigParser + +from utilities.drivers.hbpIncf import * +from utilities.drivers.euhit import * +from utilities.drivers.myproxy import * +from utilities.drivers.eudatunity import * + +logging.basicConfig() + +class SyncRemoteUsers: + + def __init__(self): + """initialize the object""" + + self.logger = logging.getLogger('remote.users.sync') + + + def main(self): + """ + It synchronizes the remote users accounts with a local json file + """ + + parser = argparse.ArgumentParser(description='Synchronize remote user ' + 'accounts to a local json ' + 'file.') + parser.add_argument('-r', '--remove', action='store_true', dest='remove', + default=False, help='remove users and groups that do' + ' not exist in the remote domain') + parser.add_argument('-d', '--debug', action='store_true', + dest='debug', default=False, + help='print debug messages') + parser.add_argument('conf', default='remote.users.sync.conf', + help='path to the configuration file') + + subparsers = parser.add_subparsers(title='Target group', + help='additional help') + parser_group = subparsers.add_parser('syncto', + help='the syncronization target') + parser_group.add_argument('group', help='the target group (or project)') + parser_group.add_argument('-s', '--subgroup', dest='subgroup', + default='', help='the target sub-group') + + _args = parser.parse_args() + + self.config = ConfigParser.RawConfigParser() + self.config.readfp(open(_args.conf)) + logfilepath = self._getConfOption('Common', 'logfile') + loglevel = self._getConfOption('Common', 'loglevel') + self.filepath = self._getConfOption('Common', 'usersfile') + + main_project = _args.group + subproject = _args.subgroup + remove = _args.remove + + ll = {'INFO': logging.INFO, 'DEBUG': logging.DEBUG, \ + 'ERROR': logging.ERROR, 'WARNING': logging.WARNING} + self.logger.setLevel(ll[loglevel]) + if (_args.debug): + self.logger.setLevel(logging.DEBUG) + + rfh = logging.handlers.RotatingFileHandler(logfilepath, + maxBytes=4194304, + backupCount=1) + formatter = logging.Formatter('%(asctime)s %(levelname)s:%(message)s') + rfh.setFormatter(formatter) + self.logger.addHandler(rfh) + + # Get the json file containing the list of projects, sub-groups and + # related members + with open(self.filepath, "r") as jsonFile: + data = json.load(jsonFile) + + if main_project in data: + if subproject: + if subproject not in data[main_project]["groups"]: + self.logger.error('\'' + subproject + '\' group not found.') + sys.exit(1) + else: + self.logger.error('\'' + main_project + '\' group not found.') + sys.exit(1) + + userparam = {k:v for k,v in self.config.items(main_project)} + if (main_project == 'EUDAT'): + self.logger.info('Syncronizing local json file with eudat user DB...') + eudatRemoteSource = EudatRemoteSource(userparam, self.logger) + local_users_by_org = data[main_project]["groups"] + data = eudatRemoteSource.synchronize_user_db(local_users_by_org, \ + data, remove) + + if data == None: sys.exit(1) + + with open(self.filepath, "w") as jsonFile: + jsonFile.write(json.dumps(data,indent=2)) + self.logger.info('{0} correctly written!'.format(self.filepath)) + jsonFile.close() + + sys.exit(0) + + + def _getConfOption(self, section, option): + """ + get the options from the configuration file + """ + + if (self.config.has_option(section, option)): + return self.config.get(section, option) + else: + self.logger.error('missing parameter %s:%s' % (section,option)) + sys.exit(1) + + +if __name__ == '__main__': + SyncRemoteUsers().main() diff --git a/scripts/test/irods.external b/scripts/test/irods.external new file mode 100644 index 0000000..4e918c8 --- /dev/null +++ b/scripts/test/irods.external @@ -0,0 +1,47 @@ +{ + "EUHIT_Repo": { + "members": [ + "user1", + "user2", + "user3", + "user4" + ], + "groups": { + "group1": [ + "user7", + "user4" + ], + "group2": [ + "user12" + ], + "group3": [], + "group4": [ + "user9", + "user10" + ] + } + }, + "EUDAT": { + "members": [ + "user_epos" + ], + "groups": {} + }, + "HBP_prjtome": { + "members": [], + "groups": { + "group18": [ + "user1" + ], + "group15": [ + "user6", + "user35", + "user21", + "user32", + ], + "group_one": [ + "user_one" + ] + } + } +} diff --git a/scripts/utilities/__init__.py b/scripts/utilities/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/utilities/drivers/__init__.py b/scripts/utilities/drivers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/utilities/drivers/eudatunity.py b/scripts/utilities/drivers/eudatunity.py new file mode 100644 index 0000000..941967e --- /dev/null +++ b/scripts/utilities/drivers/eudatunity.py @@ -0,0 +1,135 @@ +# !/usr/bin/env python + +import sys +import json +import urllib2 +import base64 +from pprint import pformat + +class EudatRemoteSource: + def _debugMsg(self, method, msg): + """Internal: Print a debug message if debug is enabled. + + """ + if self.debug: + print "[", method, "]", msg + + def __init__(self, conf, parent_logger=None): + """initialize the object""" + + if (parent_logger): self.logger = parent_logger + else: self.logger = logging.getLogger('eudat') + + self.main_project = 'EUDAT' + + missingp = [] + key = 'host' + if key in conf: self.host = conf[key] + else: missingp.append(key) + key = 'username' + if key in conf: self.username = conf[key] + else: missingp.append(key) + key = 'password' + if key in conf: self.password = conf[key] + else: missingp.append(key) + if len(missingp) > 0: + self.logger.error('missing parameters: ' + pformat(missingp)) + + + def queryUnity(self, sublink): + """ + :param argument: url to unitydb with entity (entityID) or group (groupName) + :return: + """ + auth = base64.encodestring('%s:%s' % (self.username, self.password))[:-1] + header = "Basic %s" % auth + url = self.host + sublink + request = urllib2.Request(url) + request.add_header("Authorization",header) + try: + response = urllib2.urlopen(request) + except IOError, e: + print "Wrong username or password" + sys.exit(1) + + assert response.code == 200 + json_data = response.read() + response_dict = json.loads(json_data) + + return response_dict + + + def getRemoteUsers(self): + """ + Get the remote users' list + """ + + self.logger.info("Getting list of users from eudat db...") + # get list of all groups in Unity + group_list = self.queryUnity("group/%2F") + + final_list = {} + list_member = [] + users_map = {} + for member_id in group_list['members']: + user_record = self.queryUnity("entity/"+str(member_id)) + for identity in user_record['identities']: + if identity['typeId'] == "userName": + list_member.append(identity['value']) + users_map[member_id] = identity['value'] +## TODO: if typeId = "persistent" get value and combine +## with eudat CA root issuer DN to build dynamically the user DN + + # Append list_member to final_list + final_list['members'] = list_member + + # Query and get list of all user from Groups in Unity + list_group = {} + for group_name in group_list['subGroups']: + member_list = self.queryUnity("group"+group_name) + user_list = [] + for member_id in member_list['members']: + user_list.append(users_map[member_id]) + list_group[group_name[1:]] = user_list + + # Append list_group to final_list + final_list['groups'] = list_group + + return final_list + + + def synchronize_user_db(self, local_users_list, data, remove=False): + """ + Synchronize the remote users' list with a local json file (user db) + """ + + remote_users_list = self.getRemoteUsers() + + for org,members in remote_users_list['groups'].iteritems(): + + #if subgroup org doesn't exist, create it + org = 'eudat_' + org + if (org not in data[self.main_project]["groups"]): + self.logger.info('Creating sub-group \''+ org + '\'') + data[self.main_project]["groups"][org] = [] + + # add new members + self.logger.info('Adding users that have been added to ' + org + ' ...') + for member in members: + member = 'eudat_' + member + if member not in data[self.main_project]["groups"][org]: + data[self.main_project]["groups"][org].append(member) + self.logger.debug('\tadded user %s' % (member,)) + + # remove users that don't exist in remote groups + if remove: + for org,members in local_users_list.iteritems(): + self.logger.info('Removing users that have been removed from ' + org + '...') + for member in members: + remote_org = org[6:] + remote_member = member[6:] + if remote_member not in remote_users_list[remote_org]: + data[self.main_project]["groups"][org].remove(member) + self.logger.debug('\tremoved user %s' % (member,)) + + return data \ No newline at end of file