Merge pull request #3 from EUDAT-B2SAFE/authentication

The merging operation will include in the master branch the initial set of authorization scripts, which allow to synchronize the EUDAT userDB (Unity) with a cache, local to the EUDAT node running the scripts, implemented through json file.

scripts/conf/remote.users.sync.conf

@@ -0,0 +1,28 @@
+# section containing the common options
+[Common]
+logfile=remote.sync.log
+# possible values: INFO, DEBUG, ERROR, WARNING
+loglevel=DEBUG
+usersfile=test/irods.external
+#dnsfile=test/irods.DNs.map
+
+# section for HBP_prjtome
+[HBP_prjtome]
+
+# section for EUHIT_Repo
+[EUHIT_Repo]
+ckan_api_root=xxxxxxxxxxxx
+admin_api_key=xxxxxxxxxxxxx
+# default_groups=proj1, proj2, proj3 ...
+# groups intended for all the users
+default_groups=EUHIT_Read
+
+# section for MyProxy
+[MyProxy]
+irods_path_to_dns=/CINECA01/home/proirod1/validDN
+gridftp_cert_dn=/C=IT/O=INFN/OU=Host/L=CINECA-SAP/CN=data.repo.cineca.it
+
+[EUDAT]
+host=https://unity.eudat-aai.fz-juelich.de:8443/rest-admin/v1/
+username=
+password=

scripts/remote.users.sync.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python
+# -*- python -*-
+
+import sys
+import argparse
+import json
+import subprocess
+import logging
+import logging.handlers 
+import urllib2
+import urllib
+from pprint import pformat
+import os
+import ConfigParser
+
+from utilities.drivers.hbpIncf import *
+from utilities.drivers.euhit import *
+from utilities.drivers.myproxy import *
+from utilities.drivers.eudatunity import *
+
+logging.basicConfig()
+
+class SyncRemoteUsers:
+
+    def __init__(self):
+        """initialize the object"""
+
+        self.logger = logging.getLogger('remote.users.sync')
+
+
+    def main(self):
+        """
+        It synchronizes the remote users accounts with a local json file
+        """
+
+        parser = argparse.ArgumentParser(description='Synchronize remote user '
+                                                     'accounts to a local json '
+                                                     'file.')
+        parser.add_argument('-r', '--remove', action='store_true', dest='remove',
+                            default=False, help='remove users and groups that do'
+                                               ' not exist in the remote domain')
+        parser.add_argument('-d', '--debug', action='store_true', 
+                            dest='debug', default=False, 
+                            help='print debug messages')
+        parser.add_argument('conf', default='remote.users.sync.conf',
+                            help='path to the configuration file')
+
+        subparsers = parser.add_subparsers(title='Target group',
+                                           help='additional help')
+        parser_group = subparsers.add_parser('syncto', 
+                                             help='the syncronization target')
+        parser_group.add_argument('group', help='the target group (or project)')
+        parser_group.add_argument('-s', '--subgroup', dest='subgroup',
+                                  default='', help='the target sub-group')
+
+        _args = parser.parse_args()
+
+        self.config = ConfigParser.RawConfigParser()
+        self.config.readfp(open(_args.conf))
+        logfilepath = self._getConfOption('Common', 'logfile')
+        loglevel = self._getConfOption('Common', 'loglevel')
+        self.filepath = self._getConfOption('Common', 'usersfile')
+
+        main_project = _args.group
+        subproject = _args.subgroup
+        remove = _args.remove
+
+        ll = {'INFO': logging.INFO, 'DEBUG': logging.DEBUG, \
+              'ERROR': logging.ERROR, 'WARNING': logging.WARNING}
+        self.logger.setLevel(ll[loglevel])
+        if (_args.debug): 
+            self.logger.setLevel(logging.DEBUG)
+
+        rfh = logging.handlers.RotatingFileHandler(logfilepath, 
+                                                   maxBytes=4194304,
+                                                   backupCount=1)
+        formatter = logging.Formatter('%(asctime)s %(levelname)s:%(message)s')
+        rfh.setFormatter(formatter)
+        self.logger.addHandler(rfh)
+
+        # Get the json file containing the list of projects, sub-groups and 
+        # related members
+        with open(self.filepath, "r") as jsonFile:
+            data = json.load(jsonFile)
+
+        if main_project in data:
+            if subproject:
+                if subproject not in data[main_project]["groups"]:
+                    self.logger.error('\'' + subproject + '\' group not found.')
+                    sys.exit(1)
+        else:
+            self.logger.error('\'' + main_project + '\' group not found.')
+            sys.exit(1)
+
+        userparam = {k:v for k,v in self.config.items(main_project)}
+        if (main_project == 'EUDAT'):
+            self.logger.info('Syncronizing local json file with eudat user DB...')
+            eudatRemoteSource = EudatRemoteSource(userparam, self.logger)
+            local_users_by_org = data[main_project]["groups"]
+            data = eudatRemoteSource.synchronize_user_db(local_users_by_org, \
+                                                         data, remove)
+
+        if data == None: sys.exit(1)
+
+        with open(self.filepath, "w") as jsonFile:
+            jsonFile.write(json.dumps(data,indent=2))
+        self.logger.info('{0} correctly written!'.format(self.filepath))
+        jsonFile.close()
+
+        sys.exit(0)
+
+
+    def _getConfOption(self, section, option):
+        """
+        get the options from the configuration file
+        """
+
+        if (self.config.has_option(section, option)):
+            return self.config.get(section, option)
+        else:
+            self.logger.error('missing parameter %s:%s' % (section,option))
+            sys.exit(1)
+
+
+if __name__ == '__main__':
+    SyncRemoteUsers().main()

scripts/test/irods.external

@@ -0,0 +1,47 @@
+{
+  "EUHIT_Repo": {
+    "members": [
+      "user1", 
+      "user2", 
+      "user3", 
+      "user4"
+    ], 
+    "groups": {
+      "group1": [
+        "user7", 
+        "user4"
+      ], 
+      "group2": [
+        "user12"
+      ], 
+      "group3": [], 
+      "group4": [
+        "user9", 
+        "user10" 
+      ] 
+    }
+  }, 
+  "EUDAT": {
+    "members": [
+      "user_epos"
+    ], 
+    "groups": {}
+  }, 
+  "HBP_prjtome": {
+    "members": [], 
+    "groups": {
+      "group18": [
+        "user1"
+      ], 
+      "group15": [
+        "user6", 
+        "user35", 
+        "user21", 
+        "user32", 
+      ], 
+      "group_one": [
+        "user_one"
+      ]
+    }
+  } 
+}

scripts/utilities/drivers/eudatunity.py

@@ -0,0 +1,135 @@
+# !/usr/bin/env python
+
+import sys
+import json
+import urllib2
+import base64
+from pprint import pformat
+
+class EudatRemoteSource:
+    def _debugMsg(self, method, msg):
+        """Internal: Print a debug message if debug is enabled.
+
+        """
+        if self.debug:
+            print "[", method, "]", msg
+
+    def __init__(self, conf, parent_logger=None):
+        """initialize the object"""
+
+        if (parent_logger): self.logger = parent_logger
+        else: self.logger = logging.getLogger('eudat')
+
+        self.main_project = 'EUDAT'
+
+        missingp = []    
+        key = 'host'
+        if key in conf: self.host = conf[key]
+        else: missingp.append(key)
+        key = 'username'
+        if key in conf: self.username = conf[key]
+        else: missingp.append(key)
+        key = 'password'
+        if key in conf: self.password = conf[key]
+        else: missingp.append(key)
+        if len(missingp) > 0:
+            self.logger.error('missing parameters: ' + pformat(missingp))
+
+
+    def queryUnity(self, sublink):
+        """
+        :param argument: url to unitydb with entity (entityID) or group (groupName)
+        :return:
+        """
+        auth = base64.encodestring('%s:%s' % (self.username, self.password))[:-1]
+        header = "Basic %s" % auth
+        url = self.host + sublink
+        request = urllib2.Request(url)
+        request.add_header("Authorization",header)
+        try:
+            response = urllib2.urlopen(request)
+        except IOError, e:
+            print "Wrong username or password"
+            sys.exit(1)
+
+        assert response.code == 200
+        json_data = response.read()
+        response_dict = json.loads(json_data)
+
+        return response_dict
+
+
+    def getRemoteUsers(self):
+        """
+        Get the remote users' list
+        """
+
+        self.logger.info("Getting list of users from eudat db...")
+        # get list of all groups in Unity
+        group_list = self.queryUnity("group/%2F")
+
+        final_list = {}
+        list_member = []
+        users_map = {}
+        for member_id in group_list['members']:
+            user_record = self.queryUnity("entity/"+str(member_id))
+            for identity in user_record['identities']:
+                if identity['typeId'] == "userName":
+                    list_member.append(identity['value'])
+                    users_map[member_id] = identity['value']
+## TODO: if typeId = "persistent" get value and combine 
+##       with eudat CA root issuer DN to build dynamically the user DN
+
+        # Append list_member to final_list
+        final_list['members'] = list_member
+
+        # Query and get list of all user from Groups in Unity
+        list_group = {}
+        for group_name in group_list['subGroups']:
+            member_list = self.queryUnity("group"+group_name)
+            user_list = []
+            for member_id in member_list['members']:
+                user_list.append(users_map[member_id])
+            list_group[group_name[1:]] = user_list
+
+        # Append list_group to final_list
+        final_list['groups'] = list_group
+
+        return final_list
+
+
+    def synchronize_user_db(self, local_users_list, data, remove=False):
+        """
+        Synchronize the remote users' list with a local json file (user db)
+        """
+
+        remote_users_list = self.getRemoteUsers()
+
+        for org,members in remote_users_list['groups'].iteritems():
+
+            #if subgroup org doesn't exist, create it
+            org = 'eudat_' + org
+            if (org not in data[self.main_project]["groups"]):
+                self.logger.info('Creating sub-group \''+ org + '\'')
+                data[self.main_project]["groups"][org] = []
+
+            # add new members
+            self.logger.info('Adding users that have been added to ' + org + ' ...')
+            for member in members:
+                member = 'eudat_' + member
+                if member not in data[self.main_project]["groups"][org]:
+                    data[self.main_project]["groups"][org].append(member)
+                    self.logger.debug('\tadded user %s' % (member,))
+
+        # remove users that don't exist in remote groups
+        if remove:
+            for org,members in local_users_list.iteritems():
+                self.logger.info('Removing users that have been removed from ' + org + '...') 
+                for member in members:
+                    remote_org = org[6:]
+                    remote_member = member[6:]
+                    if remote_member not in remote_users_list[remote_org]:
+                        data[self.main_project]["groups"][org].remove(member)
+                        self.logger.debug('\tremoved user %s' % (member,))
+
+        return data