From 409a807d34b7a4edd60965c013bf922de270bf08 Mon Sep 17 00:00:00 2001 From: Chun Ly Date: Tue, 13 Oct 2020 12:59:53 -0700 Subject: [PATCH 01/11] FigshareInstituteAdmin: Add flag keyword option in FigshareInstituteAdmin #107 --- ldcoolp/curation/api/figshare.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/ldcoolp/curation/api/figshare.py b/ldcoolp/curation/api/figshare.py index e856425a..35275844 100644 --- a/ldcoolp/curation/api/figshare.py +++ b/ldcoolp/curation/api/figshare.py @@ -183,6 +183,7 @@ def get_account_list(self): accounts_df = pd.DataFrame(accounts) accounts_df = accounts_df.drop(columns='institution_id') + return accounts_df def get_account_group_roles(self, account_id): @@ -192,7 +193,7 @@ def get_account_group_roles(self, account_id): roles = issue_request('GET', url, self.headers) return roles - def get_account_details(self): + def get_account_details(self, flag=True): """ Retrieve account details. This includes number of articles, projects, collections, group association, and administrative and reviewer flags @@ -200,6 +201,7 @@ def get_account_details(self): # Retrieve accounts accounts_df = self.get_account_list() + n_accounts = accounts_df.shape[0] # Retrieve groups @@ -209,8 +211,9 @@ def get_account_details(self): num_projects = np.zeros(n_accounts, dtype=np.int) num_collections = np.zeros(n_accounts, dtype=np.int) - admin_flag = [''] * n_accounts - reviewer_flag = [''] * n_accounts + if flag: + admin_flag = [''] * n_accounts + reviewer_flag = [''] * n_accounts group_assoc = ['N/A'] * n_accounts # Determine group roles for each account @@ -237,19 +240,21 @@ def get_account_details(self): for key in roles.keys(): for t_dict in roles[key]: - if t_dict['id'] == 2: - admin_flag[n] = 'X' - if t_dict['id'] == 49: - reviewer_flag[n] = 'X' if t_dict['id'] == 11: group_assoc[n] = key + if flag: + if t_dict['id'] == 2: + admin_flag[n] = 'X' + if t_dict['id'] == 49: + reviewer_flag[n] = 'X' accounts_df['Articles'] = num_articles accounts_df['Projects'] = num_projects accounts_df['Collections'] = num_collections - accounts_df['Admin'] = admin_flag - accounts_df['Reviewer'] = reviewer_flag + if flag: + accounts_df['Admin'] = admin_flag + accounts_df['Reviewer'] = reviewer_flag for group_id, group_name in zip(groups_df['id'], groups_df['name']): self.log.info(f"{group_id} - {group_name}") From e7441924c14ae6cd2a7650a533d3f86350fcc98b Mon Sep 17 00:00:00 2001 From: Chun Ly Date: Tue, 13 Oct 2020 13:01:54 -0700 Subject: [PATCH 02/11] FigshareInstituteAdmin: Add ignore_admin option in get_account_list #107 --- ldcoolp/curation/api/figshare.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ldcoolp/curation/api/figshare.py b/ldcoolp/curation/api/figshare.py index 35275844..f93bffa0 100644 --- a/ldcoolp/curation/api/figshare.py +++ b/ldcoolp/curation/api/figshare.py @@ -173,7 +173,7 @@ def get_groups(self): groups_df = pd.DataFrame(groups) return groups_df - def get_account_list(self): + def get_account_list(self, ignore_admin=False): """Retrieve accounts within institutional instance""" url = self.endpoint("accounts") @@ -184,6 +184,14 @@ def get_account_list(self): accounts_df = pd.DataFrame(accounts) accounts_df = accounts_df.drop(columns='institution_id') + if ignore_admin: + self.log.info("Excluding administrative and test accounts") + + drop_index = list(accounts_df[accounts_df['email'] == + 'data-management@email.arizona.edu'].index) + drop_index += list(accounts_df[accounts_df['email'].str.contains('-test@email.arizona.edu')].index) + + accounts_df = accounts_df.drop(drop_index) return accounts_df def get_account_group_roles(self, account_id): @@ -193,14 +201,14 @@ def get_account_group_roles(self, account_id): roles = issue_request('GET', url, self.headers) return roles - def get_account_details(self, flag=True): + def get_account_details(self, flag=True, ignore_admin=False): """ Retrieve account details. This includes number of articles, projects, collections, group association, and administrative and reviewer flags """ # Retrieve accounts - accounts_df = self.get_account_list() + accounts_df = self.get_account_list(ignore_admin=ignore_admin) n_accounts = accounts_df.shape[0] From 92b0df348051f46b4dedcdb1889f2b10e4a152de Mon Sep 17 00:00:00 2001 From: Chun Ly Date: Tue, 13 Oct 2020 13:04:38 -0700 Subject: [PATCH 03/11] Add get_user_details script #107 --- ldcoolp/scripts/get_user_details | 80 ++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 ldcoolp/scripts/get_user_details diff --git a/ldcoolp/scripts/get_user_details b/ldcoolp/scripts/get_user_details new file mode 100755 index 00000000..c5501ec6 --- /dev/null +++ b/ldcoolp/scripts/get_user_details @@ -0,0 +1,80 @@ +#!/usr/bin/env python + +from os.path import dirname, exists, join +from os import mkdir, stat + +import argparse + +from datetime import date + +from ldcoolp.curation.api.figshare import FigshareInstituteAdmin +from ldcoolp.logger import LogClass, get_user_hostname +from ldcoolp.admin import permissions + +# Version and branch info +from ldcoolp import __version__ +from ldcoolp.git_info import get_active_branch_name, get_latest_commit +from ldcoolp import __file__ as library_path + +today = date.today() + +# Config loader +from ldcoolp.config import dict_load + +library_root_path = dirname(dirname(library_path)) # Retrieve parent directory to ldcoolp + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Command-line driver for LD-Cool-P prerequisite set-up.') + parser.add_argument('--config', required=True, help='path to configuration file') + parser.add_argument('--write_file', action='store_true', help='Write CSV file containing results') + args = parser.parse_args() + + if not exists(args.config): + raise FileNotFoundError(f"WARNING!!! Config file not found: {args.config}") + + branch_name = get_active_branch_name(library_root_path) + git_commit, git_short_commit = get_latest_commit(library_root_path) + + # Load configuration + config_dict = dict_load(args.config) + + curation_dict = config_dict['curation'] + + # Define logfile + root_directory_main = curation_dict[curation_dict['log_parent_dir']] + + log_dir = join(root_directory_main, curation_dict['log_dir']) + if not exists(log_dir): + mkdir(log_dir) + logfile_prefix = 'get_user_details' + logfile = f"{logfile_prefix}.{today.strftime('%Y-%m-%d')}.log" + + log = LogClass(log_dir, logfile).get_logger() + + log.info("****************************") + + log.debug(f"LD-Cool-P branch: {branch_name}") + log.debug(f"LD-Cool-P version: {__version__} ({git_short_commit})") + log.debug(f"LD-Cool-P commit hash: {git_commit}") + + # Retrieve username, hostname, IP + sys_info = get_user_hostname() + log.debug(f"username : {sys_info['user']}") + log.debug(f"hostname : {sys_info['hostname']}") + log.debug(f"IP Addr : {sys_info['ip']}") + log.debug(f"Op. Sys. : {sys_info['os']}") + + fs_admin = FigshareInstituteAdmin(figshare_dict=config_dict['figshare'], + log=log) + + accounts_df = fs_admin.get_account_details(flag=False, ignore_admin=True) + + log.info(f"Number of users: {len(accounts_df)}") + if not args.write_file: + print(accounts_df) + else: + log.info("Writing file") + + log.info("****************************") + log.info("Exit 0") From ea59f7c050d78dc1ca3ab3418d474629fb654c68 Mon Sep 17 00:00:00 2001 From: Chun Ly Date: Tue, 13 Oct 2020 13:24:49 -0700 Subject: [PATCH 04/11] FigshareInstituteAdmin: Reset index of df from get_account_list #107 --- ldcoolp/curation/api/figshare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ldcoolp/curation/api/figshare.py b/ldcoolp/curation/api/figshare.py index f93bffa0..a4dbea46 100644 --- a/ldcoolp/curation/api/figshare.py +++ b/ldcoolp/curation/api/figshare.py @@ -191,7 +191,7 @@ def get_account_list(self, ignore_admin=False): 'data-management@email.arizona.edu'].index) drop_index += list(accounts_df[accounts_df['email'].str.contains('-test@email.arizona.edu')].index) - accounts_df = accounts_df.drop(drop_index) + accounts_df = accounts_df.drop(drop_index).reset_index(drop=True) return accounts_df def get_account_group_roles(self, account_id): From f8af5d19cb55166f9e7e256f0abc7a1b76f60e74 Mon Sep 17 00:00:00 2001 From: Chun Ly Date: Tue, 13 Oct 2020 13:25:59 -0700 Subject: [PATCH 05/11] get_user_details: Write file when --write_file set #107 --- ldcoolp/scripts/get_user_details | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/ldcoolp/scripts/get_user_details b/ldcoolp/scripts/get_user_details index c5501ec6..d72ccb27 100755 --- a/ldcoolp/scripts/get_user_details +++ b/ldcoolp/scripts/get_user_details @@ -5,7 +5,8 @@ from os import mkdir, stat import argparse -from datetime import date +from datetime import date, datetime +import pytz from ldcoolp.curation.api.figshare import FigshareInstituteAdmin from ldcoolp.logger import LogClass, get_user_hostname @@ -16,11 +17,14 @@ from ldcoolp import __version__ from ldcoolp.git_info import get_active_branch_name, get_latest_commit from ldcoolp import __file__ as library_path -today = date.today() - # Config loader from ldcoolp.config import dict_load +today = date.today() + +tz_AZ = pytz.timezone('US/Arizona') +now = datetime.now(tz_AZ) + library_root_path = dirname(dirname(library_path)) # Retrieve parent directory to ldcoolp @@ -74,7 +78,10 @@ if __name__ == '__main__': if not args.write_file: print(accounts_df) else: - log.info("Writing file") + csv_outfile = f"redata_user_details.{now.strftime('%Y-%m-%d_%H:%M')}.log" + log.info(f"Writing file : {csv_outfile}") + + accounts_df.to_csv(csv_outfile) log.info("****************************") log.info("Exit 0") From 6f3c595268b0eb6f47fbd22be2b56783c5883cfc Mon Sep 17 00:00:00 2001 From: Chun Ly Date: Tue, 13 Oct 2020 13:26:27 -0700 Subject: [PATCH 06/11] get_user_details: File permission changes #107 --- ldcoolp/scripts/get_user_details | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ldcoolp/scripts/get_user_details b/ldcoolp/scripts/get_user_details index d72ccb27..4570ab24 100755 --- a/ldcoolp/scripts/get_user_details +++ b/ldcoolp/scripts/get_user_details @@ -83,5 +83,13 @@ if __name__ == '__main__': accounts_df.to_csv(csv_outfile) + # Change permission to mode=666 (rw for all) + status = stat(join(log_dir, logfile)) + if oct(status.st_mode)[-3:] == '666': + log.debug("Permissions set for logfile") + else: + log.debug("Changing permissions on logfile...") + permissions.curation(join(log_dir, logfile), mode=0o666) + log.info("****************************") log.info("Exit 0") From a4ac56eee5c09118c002237bc323074998a47197 Mon Sep 17 00:00:00 2001 From: Chun Ly Date: Tue, 13 Oct 2020 14:19:22 -0700 Subject: [PATCH 07/11] get_user_details: Change suffix for csv_outfile #107 --- ldcoolp/scripts/get_user_details | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ldcoolp/scripts/get_user_details b/ldcoolp/scripts/get_user_details index 4570ab24..e59c37dc 100755 --- a/ldcoolp/scripts/get_user_details +++ b/ldcoolp/scripts/get_user_details @@ -78,7 +78,7 @@ if __name__ == '__main__': if not args.write_file: print(accounts_df) else: - csv_outfile = f"redata_user_details.{now.strftime('%Y-%m-%d_%H:%M')}.log" + csv_outfile = f"redata_user_details.{now.strftime('%Y-%m-%d_%H:%M')}.csv" log.info(f"Writing file : {csv_outfile}") accounts_df.to_csv(csv_outfile) From 815735166f47b6735819afa0d7691fefe40f0aa9 Mon Sep 17 00:00:00 2001 From: Chun Ly Date: Tue, 13 Oct 2020 14:25:25 -0700 Subject: [PATCH 08/11] get_user_details: Add path option for csv_outfile #107 Turn on indexing in written CSV file --- ldcoolp/scripts/get_user_details | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ldcoolp/scripts/get_user_details b/ldcoolp/scripts/get_user_details index e59c37dc..c2cdcae8 100755 --- a/ldcoolp/scripts/get_user_details +++ b/ldcoolp/scripts/get_user_details @@ -32,11 +32,15 @@ if __name__ == '__main__': parser = argparse.ArgumentParser(description='Command-line driver for LD-Cool-P prerequisite set-up.') parser.add_argument('--config', required=True, help='path to configuration file') parser.add_argument('--write_file', action='store_true', help='Write CSV file containing results') + parser.add_argument('--path', required=True, help='Full path to write CSV file') args = parser.parse_args() if not exists(args.config): raise FileNotFoundError(f"WARNING!!! Config file not found: {args.config}") + if not exists(args.path): + raise FileNotFoundError(f"WARNING!!! Path not found: {args.path}") + branch_name = get_active_branch_name(library_root_path) git_commit, git_short_commit = get_latest_commit(library_root_path) @@ -78,10 +82,12 @@ if __name__ == '__main__': if not args.write_file: print(accounts_df) else: - csv_outfile = f"redata_user_details.{now.strftime('%Y-%m-%d_%H:%M')}.csv" + csv_outfile = join(args.path, + f"redata_user_details.{now.strftime('%Y-%m-%d_%H:%M')}.csv") + log.info(f"Writing file : {csv_outfile}") - accounts_df.to_csv(csv_outfile) + accounts_df.to_csv(csv_outfile, index=False) # Change permission to mode=666 (rw for all) status = stat(join(log_dir, logfile)) From cca534823b86488335ef4680201ddbf0127c4c0e Mon Sep 17 00:00:00 2001 From: Chun Ly Date: Tue, 13 Oct 2020 14:34:17 -0700 Subject: [PATCH 09/11] get_user_details: Add --simple option to get retrieve list #107 Set --path option as required with --write_file --- ldcoolp/scripts/get_user_details | 33 ++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/ldcoolp/scripts/get_user_details b/ldcoolp/scripts/get_user_details index c2cdcae8..77d78b17 100755 --- a/ldcoolp/scripts/get_user_details +++ b/ldcoolp/scripts/get_user_details @@ -32,14 +32,17 @@ if __name__ == '__main__': parser = argparse.ArgumentParser(description='Command-line driver for LD-Cool-P prerequisite set-up.') parser.add_argument('--config', required=True, help='path to configuration file') parser.add_argument('--write_file', action='store_true', help='Write CSV file containing results') - parser.add_argument('--path', required=True, help='Full path to write CSV file') + parser.add_argument('--path', help='Full path to write CSV file') + parser.add_argument('--simple', action='store_true', + help='Generate a basic list without detailed information') args = parser.parse_args() if not exists(args.config): raise FileNotFoundError(f"WARNING!!! Config file not found: {args.config}") - if not exists(args.path): - raise FileNotFoundError(f"WARNING!!! Path not found: {args.path}") + if args.write_file: + if not exists(args.path): + raise FileNotFoundError(f"WARNING!!! Path not found: {args.path}") branch_name = get_active_branch_name(library_root_path) git_commit, git_short_commit = get_latest_commit(library_root_path) @@ -76,18 +79,24 @@ if __name__ == '__main__': fs_admin = FigshareInstituteAdmin(figshare_dict=config_dict['figshare'], log=log) - accounts_df = fs_admin.get_account_details(flag=False, ignore_admin=True) + if not args.simple: + accounts_df = fs_admin.get_account_details(flag=False, ignore_admin=True) - log.info(f"Number of users: {len(accounts_df)}") - if not args.write_file: - print(accounts_df) - else: - csv_outfile = join(args.path, - f"redata_user_details.{now.strftime('%Y-%m-%d_%H:%M')}.csv") + log.info(f"Number of users: {len(accounts_df)}") + if not args.write_file: + print(accounts_df) + else: + csv_outfile = join(args.path, + f"redata_user_details.{now.strftime('%Y-%m-%d_%H:%M')}.csv") - log.info(f"Writing file : {csv_outfile}") + log.info(f"Writing file : {csv_outfile}") - accounts_df.to_csv(csv_outfile, index=False) + accounts_df.to_csv(csv_outfile, index=False) + else: + accounts_df = fs_admin.get_account_list(ignore_admin=True) + + log.info(f"Number of users: {len(accounts_df)}") + print(accounts_df) # Change permission to mode=666 (rw for all) status = stat(join(log_dir, logfile)) From 4c60d2850842c38fd985d14e5d868282168827bc Mon Sep 17 00:00:00 2001 From: Chun Ly Date: Tue, 13 Oct 2020 14:39:38 -0700 Subject: [PATCH 10/11] Bump version: v0.15.1 -> v0.15.2 #107 --- README.md | 2 +- ldcoolp/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 646f7ad1..fe683ade 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ You can confirm installation via `conda list` (curation) $ conda list ldcoolp ``` -You should see that the version is `0.15.1`. +You should see that the version is `0.15.2`. ### Configuration Settings diff --git a/ldcoolp/__init__.py b/ldcoolp/__init__.py index 8a7958de..dd5d55ca 100644 --- a/ldcoolp/__init__.py +++ b/ldcoolp/__init__.py @@ -1,6 +1,6 @@ from os import path -__version__ = "0.15.1" +__version__ = "0.15.2" co_path = path.dirname(__file__) diff --git a/setup.py b/setup.py index e494d1db..eea06f4d 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='ldcoolp', - version='v0.15.1', + version='v0.15.2', packages=['ldcoolp'], url='https://github.com/ualibraries/LD_Cool_P', license='MIT License', From e6eb46d9170f621c18f3721573dfeb9bcbcef215 Mon Sep 17 00:00:00 2001 From: Chun Ly Date: Tue, 13 Oct 2020 14:47:18 -0700 Subject: [PATCH 11/11] Update README.md changelog #107 --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fe683ade..94796b13 100644 --- a/README.md +++ b/README.md @@ -199,11 +199,12 @@ We use [SemVer](http://semver.org/) for versioning. For the versions available, A list of released features and their issue number(s). List is sorted from moderate to minor revisions for reach release. -v0.15.0 - v0.15.1: +v0.15.0 - v0.15.2: * Implementation of Qualtrics README file #98 * Folder re-structuring for versioning #100 * `perform_move` script to perform move to next curation stage, backwards or to publish #105 + * `get_user_details` script to retrieve user information from Figshare API #107 v0.14.0 - 0.14.1: * Full stdout and file logging #83