From 4f52f20848839bac5928061763fcffc9469fe9bc Mon Sep 17 00:00:00 2001 From: Alejandro Roiz Walss Date: Tue, 27 Aug 2024 14:10:51 -0600 Subject: [PATCH 01/13] add logging abstraction --- VERSION | 2 +- confidant/app.py | 12 ++++-------- confidant/logging.py | 30 ++++++++++++++++++++++++++++++ confidant/routes/credentials.py | 8 ++++++-- confidant/settings.py | 4 ++++ 5 files changed, 45 insertions(+), 11 deletions(-) create mode 100644 confidant/logging.py diff --git a/VERSION b/VERSION index 09a7391e..f6923f94 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6.6.1 +6.6.1-alpha-1 diff --git a/confidant/app.py b/confidant/app.py index c65fc8c3..78d675ec 100644 --- a/confidant/app.py +++ b/confidant/app.py @@ -1,11 +1,9 @@ -import logging - -import boto3 import guard from flask import Flask from flask_sslify import SSLify from confidant import settings +from confidant.utils import misc from confidant.routes import ( blind_credentials, certificates, @@ -17,11 +15,6 @@ jwks, ) -if not settings.get('DEBUG'): - boto3.set_stream_logger(level=logging.CRITICAL) - logging.getLogger('botocore').setLevel(logging.CRITICAL) - logging.getLogger('pynamodb').setLevel(logging.WARNING) - CSP_POLICY = { 'default-src': ["'self'"], 'style-src': [ @@ -44,6 +37,9 @@ def create_app(): app.wsgi_app = guard.ContentSecurityPolicy(app.wsgi_app, CSP_POLICY) + init_logging_func = misc.load_module(settings.get('INIT_LOGGING_MODULE')) + init_logging_func() + if settings.REDIS_URL_FLASK_SESSIONS: import redis from flask_session import Session diff --git a/confidant/logging.py b/confidant/logging.py new file mode 100644 index 00000000..1cd4e20e --- /dev/null +++ b/confidant/logging.py @@ -0,0 +1,30 @@ +import logging +import boto3 + +from confidant import settings + +def init_logging(): + logging.getLogger(__name__).info('Initializing logging') + if not settings.get('DEBUG'): + boto3.set_stream_logger(level=logging.CRITICAL) + logging.getLogger('botocore').setLevel(logging.CRITICAL) + logging.getLogger('pynamodb').setLevel(logging.WARNING) + + +def get_logger(name=__name__): + return logging.getLogger(name) + + +def logging_abstraction(log_level='INFO', msg='', name=__name__): + logger = get_logger(name) + + if log_level == 'INFO' and msg: + logger.info(msg) + elif log_level == 'ERROR' and msg: + logger.error(msg) + elif log_level == 'DEBUG' and msg: + logger.debug(msg) + elif log_level == 'CRITICAL' and msg: + logger.critical(msg) + elif log_level == 'WARNING' and msg: + logger.warning(msg) diff --git a/confidant/routes/credentials.py b/confidant/routes/credentials.py index 023131c5..a6e70aa0 100644 --- a/confidant/routes/credentials.py +++ b/confidant/routes/credentials.py @@ -32,6 +32,8 @@ blueprint = blueprints.Blueprint('credentials', __name__) acl_module_check = misc.load_module(settings.ACL_MODULE) +logging_module = misc.load_module(settings.get('LOGGING_MODULE')) + VALUE_LENGTH = 50 @@ -203,8 +205,10 @@ def get_credential(id): try: credential = Credential.get(id) except DoesNotExist: - logger.warning( - 'Item with id {0} does not exist.'.format(id) + logging_module( + log_level='WARNING', + msg='Item with id {0} does not exist.'.format(id), + name=__name__, ) return jsonify({}), 404 if credential.data_type != 'credential': diff --git a/confidant/settings.py b/confidant/settings.py index 83a8d9c2..99195c58 100644 --- a/confidant/settings.py +++ b/confidant/settings.py @@ -691,3 +691,7 @@ def get(name, default=None): # Module that will perform an external ACL check on API endpoints ACL_MODULE = str_env('ACL_MODULE', 'confidant.authnz.rbac:default_acl') + +# Logging +INIT_LOGGING_MODULE = str_env('LOGGING_MODULE', 'confidant.logging:init_logging') +LOGGING_MODULE = str_env('LOGGING_MODULE', 'confidant.logging:logging_abstraction') From 943ca066878022ddc4f804615bf83e427ecfc36c Mon Sep 17 00:00:00 2001 From: Alejandro Roiz Walss Date: Tue, 27 Aug 2024 14:17:40 -0600 Subject: [PATCH 02/13] add log msg --- confidant/routes/credentials.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/confidant/routes/credentials.py b/confidant/routes/credentials.py index a6e70aa0..84a35313 100644 --- a/confidant/routes/credentials.py +++ b/confidant/routes/credentials.py @@ -31,6 +31,8 @@ logger = logging.getLogger(__name__) blueprint = blueprints.Blueprint('credentials', __name__) +logger.warning('Will load logging module: {}'.format(settings.get('LOGGING_MODULE'))) + acl_module_check = misc.load_module(settings.ACL_MODULE) logging_module = misc.load_module(settings.get('LOGGING_MODULE')) From 70fbe3c226ff90199231b3ec2b301be65d278f07 Mon Sep 17 00:00:00 2001 From: Alejandro Roiz Walss Date: Tue, 27 Aug 2024 15:05:29 -0600 Subject: [PATCH 03/13] fix pre-commit --- confidant/logging.py | 1 + confidant/routes/credentials.py | 4 +++- confidant/settings.py | 8 ++++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/confidant/logging.py b/confidant/logging.py index 1cd4e20e..0acaea0e 100644 --- a/confidant/logging.py +++ b/confidant/logging.py @@ -3,6 +3,7 @@ from confidant import settings + def init_logging(): logging.getLogger(__name__).info('Initializing logging') if not settings.get('DEBUG'): diff --git a/confidant/routes/credentials.py b/confidant/routes/credentials.py index 84a35313..02a6221e 100644 --- a/confidant/routes/credentials.py +++ b/confidant/routes/credentials.py @@ -31,7 +31,9 @@ logger = logging.getLogger(__name__) blueprint = blueprints.Blueprint('credentials', __name__) -logger.warning('Will load logging module: {}'.format(settings.get('LOGGING_MODULE'))) +logger.warning( + 'Will load logging module: {}'.format(settings.get('LOGGING_MODULE')) +) acl_module_check = misc.load_module(settings.ACL_MODULE) logging_module = misc.load_module(settings.get('LOGGING_MODULE')) diff --git a/confidant/settings.py b/confidant/settings.py index 99195c58..e4b41d83 100644 --- a/confidant/settings.py +++ b/confidant/settings.py @@ -693,5 +693,9 @@ def get(name, default=None): ACL_MODULE = str_env('ACL_MODULE', 'confidant.authnz.rbac:default_acl') # Logging -INIT_LOGGING_MODULE = str_env('LOGGING_MODULE', 'confidant.logging:init_logging') -LOGGING_MODULE = str_env('LOGGING_MODULE', 'confidant.logging:logging_abstraction') +INIT_LOGGING_MODULE = str_env( + 'LOGGING_MODULE', 'confidant.logging:init_logging' +) +LOGGING_MODULE = str_env( + 'LOGGING_MODULE', 'confidant.logging:logging_abstraction' +) From 4d03fa023bcfc37f83d8218846311e0e675f81d4 Mon Sep 17 00:00:00 2001 From: Alejandro Roiz Walss Date: Tue, 27 Aug 2024 21:07:35 -0600 Subject: [PATCH 04/13] KISS --- confidant/app.py | 12 +++++++++--- confidant/logging.py | 31 ------------------------------- confidant/routes/credentials.py | 15 ++++----------- confidant/settings.py | 7 +------ 4 files changed, 14 insertions(+), 51 deletions(-) delete mode 100644 confidant/logging.py diff --git a/confidant/app.py b/confidant/app.py index 78d675ec..3a0d3493 100644 --- a/confidant/app.py +++ b/confidant/app.py @@ -1,3 +1,6 @@ +import logging + +import boto3 import guard from flask import Flask from flask_sslify import SSLify @@ -15,6 +18,12 @@ jwks, ) + +if not settings.get('DEBUG'): + boto3.set_stream_logger(level=logging.CRITICAL) + logging.getLogger('botocore').setLevel(logging.CRITICAL) + logging.getLogger('pynamodb').setLevel(logging.WARNING) + CSP_POLICY = { 'default-src': ["'self'"], 'style-src': [ @@ -37,9 +46,6 @@ def create_app(): app.wsgi_app = guard.ContentSecurityPolicy(app.wsgi_app, CSP_POLICY) - init_logging_func = misc.load_module(settings.get('INIT_LOGGING_MODULE')) - init_logging_func() - if settings.REDIS_URL_FLASK_SESSIONS: import redis from flask_session import Session diff --git a/confidant/logging.py b/confidant/logging.py deleted file mode 100644 index 0acaea0e..00000000 --- a/confidant/logging.py +++ /dev/null @@ -1,31 +0,0 @@ -import logging -import boto3 - -from confidant import settings - - -def init_logging(): - logging.getLogger(__name__).info('Initializing logging') - if not settings.get('DEBUG'): - boto3.set_stream_logger(level=logging.CRITICAL) - logging.getLogger('botocore').setLevel(logging.CRITICAL) - logging.getLogger('pynamodb').setLevel(logging.WARNING) - - -def get_logger(name=__name__): - return logging.getLogger(name) - - -def logging_abstraction(log_level='INFO', msg='', name=__name__): - logger = get_logger(name) - - if log_level == 'INFO' and msg: - logger.info(msg) - elif log_level == 'ERROR' and msg: - logger.error(msg) - elif log_level == 'DEBUG' and msg: - logger.debug(msg) - elif log_level == 'CRITICAL' and msg: - logger.critical(msg) - elif log_level == 'WARNING' and msg: - logger.warning(msg) diff --git a/confidant/routes/credentials.py b/confidant/routes/credentials.py index 02a6221e..e5821acf 100644 --- a/confidant/routes/credentials.py +++ b/confidant/routes/credentials.py @@ -1,6 +1,6 @@ import base64 +import importlib import json -import logging import re import uuid @@ -28,15 +28,12 @@ from confidant.utils import maintenance, misc, stats from confidant.utils.dynamodb import decode_last_evaluated_key +logging = importlib.import_module(settings.LOGGING_MODULE) + logger = logging.getLogger(__name__) blueprint = blueprints.Blueprint('credentials', __name__) -logger.warning( - 'Will load logging module: {}'.format(settings.get('LOGGING_MODULE')) -) - acl_module_check = misc.load_module(settings.ACL_MODULE) -logging_module = misc.load_module(settings.get('LOGGING_MODULE')) VALUE_LENGTH = 50 @@ -209,11 +206,7 @@ def get_credential(id): try: credential = Credential.get(id) except DoesNotExist: - logging_module( - log_level='WARNING', - msg='Item with id {0} does not exist.'.format(id), - name=__name__, - ) + logger.warning('Item with id {0} does not exist.'.format(id)) return jsonify({}), 404 if credential.data_type != 'credential': return jsonify({}), 404 diff --git a/confidant/settings.py b/confidant/settings.py index e4b41d83..c0b157d0 100644 --- a/confidant/settings.py +++ b/confidant/settings.py @@ -693,9 +693,4 @@ def get(name, default=None): ACL_MODULE = str_env('ACL_MODULE', 'confidant.authnz.rbac:default_acl') # Logging -INIT_LOGGING_MODULE = str_env( - 'LOGGING_MODULE', 'confidant.logging:init_logging' -) -LOGGING_MODULE = str_env( - 'LOGGING_MODULE', 'confidant.logging:logging_abstraction' -) +LOGGING_MODULE = str_env('LOGGING_MODULE', 'logging') From 541cac2ae949f1327d64552fd45b830df1879e30 Mon Sep 17 00:00:00 2001 From: Alejandro Roiz Walss Date: Tue, 27 Aug 2024 21:08:11 -0600 Subject: [PATCH 05/13] remove --- confidant/app.py | 2 -- confidant/routes/credentials.py | 1 - 2 files changed, 3 deletions(-) diff --git a/confidant/app.py b/confidant/app.py index 3a0d3493..c65fc8c3 100644 --- a/confidant/app.py +++ b/confidant/app.py @@ -6,7 +6,6 @@ from flask_sslify import SSLify from confidant import settings -from confidant.utils import misc from confidant.routes import ( blind_credentials, certificates, @@ -18,7 +17,6 @@ jwks, ) - if not settings.get('DEBUG'): boto3.set_stream_logger(level=logging.CRITICAL) logging.getLogger('botocore').setLevel(logging.CRITICAL) diff --git a/confidant/routes/credentials.py b/confidant/routes/credentials.py index e5821acf..714f4877 100644 --- a/confidant/routes/credentials.py +++ b/confidant/routes/credentials.py @@ -34,7 +34,6 @@ blueprint = blueprints.Blueprint('credentials', __name__) acl_module_check = misc.load_module(settings.ACL_MODULE) - VALUE_LENGTH = 50 From dc1d454cc8903dbc36ae63ae8349fc6a1b567543 Mon Sep 17 00:00:00 2001 From: Alejandro Roiz Walss Date: Tue, 27 Aug 2024 21:08:33 -0600 Subject: [PATCH 06/13] remove --- confidant/routes/credentials.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/confidant/routes/credentials.py b/confidant/routes/credentials.py index 714f4877..083f0342 100644 --- a/confidant/routes/credentials.py +++ b/confidant/routes/credentials.py @@ -205,7 +205,9 @@ def get_credential(id): try: credential = Credential.get(id) except DoesNotExist: - logger.warning('Item with id {0} does not exist.'.format(id)) + logger.warning( + 'Item with id {0} does not exist.'.format(id) + ) return jsonify({}), 404 if credential.data_type != 'credential': return jsonify({}), 404 From d0a6a59b87c5c69cc9e12d36c60eb35429aeffed Mon Sep 17 00:00:00 2001 From: Alejandro Roiz Walss Date: Tue, 27 Aug 2024 21:08:57 -0600 Subject: [PATCH 07/13] version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f6923f94..68ab7363 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6.6.1-alpha-1 +6.6.1-alpha-2 From b1f6c824edbbe159d513d2e445c687eab7a8e674 Mon Sep 17 00:00:00 2001 From: Alejandro Roiz Walss Date: Thu, 29 Aug 2024 15:20:10 -0600 Subject: [PATCH 08/13] change all loggers to be loaded via the env var value --- CHANGELOG.md | 5 +++++ VERSION | 2 +- confidant/app.py | 4 +++- confidant/authnz/__init__.py | 3 ++- confidant/authnz/userauth.py | 3 ++- confidant/clients/__init__.py | 5 ++++- confidant/encrypted_settings.py | 4 +++- confidant/routes/blind_credentials.py | 3 ++- confidant/routes/certificates.py | 3 ++- confidant/routes/credentials.py | 1 - confidant/routes/jwks.py | 3 ++- confidant/routes/saml.py | 3 ++- confidant/routes/services.py | 3 ++- confidant/routes/static_files.py | 3 ++- confidant/scripts/archive.py | 3 ++- confidant/scripts/migrate.py | 4 +++- confidant/scripts/migrate_bool.py | 3 ++- confidant/scripts/restore.py | 3 ++- confidant/scripts/utils.py | 3 ++- confidant/services/certificatemanager.py | 3 ++- confidant/services/ciphermanager.py | 3 ++- confidant/services/credentialmanager.py | 3 ++- confidant/services/graphite.py | 3 ++- confidant/services/iamrolemanager.py | 3 ++- confidant/services/jwkmanager.py | 4 +++- confidant/services/keymanager.py | 3 ++- confidant/services/webhook.py | 3 ++- confidant/settings.py | 13 +++++++------ confidant/utils/maintenance.py | 3 ++- 29 files changed, 69 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4f783a3..581b4ad4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 6.6.2 + +* Added logging module abstraction. You can now define the logging module to be used by setting the + `LOGGING_MODULE` environment variable. + ## 6.6.1 * Upgrade confidant to python 3.10.14 diff --git a/VERSION b/VERSION index 68ab7363..28179fc1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6.6.1-alpha-2 +6.6.2 diff --git a/confidant/app.py b/confidant/app.py index c65fc8c3..f8d633b2 100644 --- a/confidant/app.py +++ b/confidant/app.py @@ -1,4 +1,4 @@ -import logging +import importlib import boto3 import guard @@ -17,6 +17,8 @@ jwks, ) +logging = importlib.import_module(settings.LOGGING_MODULE) + if not settings.get('DEBUG'): boto3.set_stream_logger(level=logging.CRITICAL) logging.getLogger('botocore').setLevel(logging.CRITICAL) diff --git a/confidant/authnz/__init__.py b/confidant/authnz/__init__.py index c19543df..0e79bf87 100644 --- a/confidant/authnz/__init__.py +++ b/confidant/authnz/__init__.py @@ -1,4 +1,4 @@ -import logging +import importlib import kmsauth from flask import abort, request, g, make_response @@ -17,6 +17,7 @@ _VALIDATOR = None +logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) user_mod = userauth.init_user_auth_class() diff --git a/confidant/authnz/userauth.py b/confidant/authnz/userauth.py index 3bc55178..22ae521e 100644 --- a/confidant/authnz/userauth.py +++ b/confidant/authnz/userauth.py @@ -1,6 +1,6 @@ import abc -import logging import datetime +import importlib import random import yaml @@ -24,6 +24,7 @@ from confidant.utils.misc import dict_deep_update from confidant.authnz import errors +logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) diff --git a/confidant/clients/__init__.py b/confidant/clients/__init__.py index 68577d81..98c243ad 100644 --- a/confidant/clients/__init__.py +++ b/confidant/clients/__init__.py @@ -1,8 +1,11 @@ """Module for accessing services external to Confidant.""" import boto3 -import logging +import importlib +from confidant import settings + +logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) CLIENT_CACHE = {} diff --git a/confidant/encrypted_settings.py b/confidant/encrypted_settings.py index b9bc7466..e6b55dee 100644 --- a/confidant/encrypted_settings.py +++ b/confidant/encrypted_settings.py @@ -1,13 +1,15 @@ import yaml import base64 -import logging +import importlib import json from cryptography.fernet import Fernet import confidant.clients +from confidant import settings from confidant.lib import cryptolib +logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) diff --git a/confidant/routes/blind_credentials.py b/confidant/routes/blind_credentials.py index 5cbf6d37..0b19f2db 100644 --- a/confidant/routes/blind_credentials.py +++ b/confidant/routes/blind_credentials.py @@ -1,4 +1,4 @@ -import logging +import importlib import uuid from flask import blueprints, jsonify, request @@ -18,6 +18,7 @@ ) from confidant.models.blind_credential import BlindCredential +logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) blueprint = blueprints.Blueprint('blind_credentials', __name__) diff --git a/confidant/routes/certificates.py b/confidant/routes/certificates.py index a5d839f0..081d5df3 100644 --- a/confidant/routes/certificates.py +++ b/confidant/routes/certificates.py @@ -1,4 +1,4 @@ -import logging +import importlib from flask import blueprints, jsonify, request @@ -15,6 +15,7 @@ ) from confidant.utils import misc +logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) blueprint = blueprints.Blueprint('certificates', __name__) diff --git a/confidant/routes/credentials.py b/confidant/routes/credentials.py index 083f0342..c1a4d69c 100644 --- a/confidant/routes/credentials.py +++ b/confidant/routes/credentials.py @@ -29,7 +29,6 @@ from confidant.utils.dynamodb import decode_last_evaluated_key logging = importlib.import_module(settings.LOGGING_MODULE) - logger = logging.getLogger(__name__) blueprint = blueprints.Blueprint('credentials', __name__) diff --git a/confidant/routes/jwks.py b/confidant/routes/jwks.py index 5737359e..c0d698ac 100644 --- a/confidant/routes/jwks.py +++ b/confidant/routes/jwks.py @@ -1,4 +1,4 @@ -import logging +import importlib from flask import blueprints, jsonify, request @@ -9,6 +9,7 @@ from confidant.settings import ACL_MODULE from confidant.utils import misc +logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) blueprint = blueprints.Blueprint('jwks', __name__) diff --git a/confidant/routes/saml.py b/confidant/routes/saml.py index e9c97f35..c00b6685 100644 --- a/confidant/routes/saml.py +++ b/confidant/routes/saml.py @@ -1,10 +1,11 @@ -import logging +import importlib import flask from flask import blueprints, jsonify, request, session from confidant import authnz, settings +logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) blueprint = blueprints.Blueprint('saml', __name__) diff --git a/confidant/routes/services.py b/confidant/routes/services.py index 9775f634..cabd6aaf 100644 --- a/confidant/routes/services.py +++ b/confidant/routes/services.py @@ -1,4 +1,4 @@ -import logging +import importlib from flask import blueprints, jsonify, request from pynamodb.exceptions import DoesNotExist, PutError @@ -23,6 +23,7 @@ from confidant.utils import maintenance, misc, stats from confidant.utils.dynamodb import decode_last_evaluated_key +logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) blueprint = blueprints.Blueprint('services', __name__) diff --git a/confidant/routes/static_files.py b/confidant/routes/static_files.py index 3ef4b372..8ddba215 100644 --- a/confidant/routes/static_files.py +++ b/confidant/routes/static_files.py @@ -1,11 +1,12 @@ import os -import logging +import importlib from flask import blueprints, current_app, send_from_directory from werkzeug.exceptions import NotFound from confidant import authnz, settings +logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) blueprint = blueprints.Blueprint('static_files', __name__) diff --git a/confidant/scripts/archive.py b/confidant/scripts/archive.py index 1f93f9e0..17d23353 100644 --- a/confidant/scripts/archive.py +++ b/confidant/scripts/archive.py @@ -1,5 +1,5 @@ import sys -import logging +import importlib from datetime import datetime from flask_script import Command, Option @@ -8,6 +8,7 @@ from confidant.models.credential import Credential from confidant.services import credentialmanager +logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler(sys.stdout)) diff --git a/confidant/scripts/migrate.py b/confidant/scripts/migrate.py index 3d60d0ad..c32142fe 100644 --- a/confidant/scripts/migrate.py +++ b/confidant/scripts/migrate.py @@ -1,7 +1,8 @@ import sys -import logging +import importlib from flask_script import Command +from confidant import settings from confidant.models.blind_credential import BlindCredential from confidant.models.service import Service @@ -11,6 +12,7 @@ from pynamodb.constants import STRING_SET from pynamodb.models import Model +logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler(sys.stdout)) logger.setLevel(logging.INFO) diff --git a/confidant/scripts/migrate_bool.py b/confidant/scripts/migrate_bool.py index c9ae6d82..f324a3c2 100644 --- a/confidant/scripts/migrate_bool.py +++ b/confidant/scripts/migrate_bool.py @@ -1,4 +1,4 @@ -import logging +import importlib import time import sys @@ -14,6 +14,7 @@ from confidant import settings +logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler(sys.stdout)) logger.setLevel(logging.INFO) diff --git a/confidant/scripts/restore.py b/confidant/scripts/restore.py index 500ad9b5..e8ce78c9 100644 --- a/confidant/scripts/restore.py +++ b/confidant/scripts/restore.py @@ -1,5 +1,5 @@ import sys -import logging +import importlib from flask_script import Command, Option from pynamodb.exceptions import DoesNotExist @@ -8,6 +8,7 @@ from confidant.models.credential import Credential, CredentialArchive from confidant.utils import stats +logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler(sys.stdout)) diff --git a/confidant/scripts/utils.py b/confidant/scripts/utils.py index c7f4ad35..cde7ecd4 100644 --- a/confidant/scripts/utils.py +++ b/confidant/scripts/utils.py @@ -1,5 +1,5 @@ import sys -import logging +import importlib from flask_script import Command from botocore.exceptions import ClientError @@ -9,6 +9,7 @@ from confidant.models.service import Service from confidant.utils.dynamodb import create_dynamodb_tables +logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler(sys.stdout)) diff --git a/confidant/services/certificatemanager.py b/confidant/services/certificatemanager.py index 8df3e552..2cc945dd 100644 --- a/confidant/services/certificatemanager.py +++ b/confidant/services/certificatemanager.py @@ -1,6 +1,6 @@ import datetime import hashlib -import logging +import importlib import time from cryptography import x509 @@ -15,6 +15,7 @@ from confidant import settings from confidant.utils import stats +logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) diff --git a/confidant/services/ciphermanager.py b/confidant/services/ciphermanager.py index 0b3624bf..160fe856 100644 --- a/confidant/services/ciphermanager.py +++ b/confidant/services/ciphermanager.py @@ -1,11 +1,12 @@ import base64 import re -import logging +import importlib from cryptography.fernet import Fernet from confidant import settings +logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) diff --git a/confidant/services/credentialmanager.py b/confidant/services/credentialmanager.py index ade09700..a2479ac0 100644 --- a/confidant/services/credentialmanager.py +++ b/confidant/services/credentialmanager.py @@ -1,6 +1,6 @@ import copy import re -import logging +import importlib from confidant import settings from confidant.models.blind_credential import BlindCredential @@ -10,6 +10,7 @@ from pynamodb.exceptions import DoesNotExist +logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) diff --git a/confidant/services/graphite.py b/confidant/services/graphite.py index d2125a73..f39a9f51 100644 --- a/confidant/services/graphite.py +++ b/confidant/services/graphite.py @@ -1,9 +1,10 @@ import requests import json -import logging +import importlib from confidant import settings +logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) diff --git a/confidant/services/iamrolemanager.py b/confidant/services/iamrolemanager.py index c38d3518..ba0d0fdb 100644 --- a/confidant/services/iamrolemanager.py +++ b/confidant/services/iamrolemanager.py @@ -1,10 +1,11 @@ import gevent -import logging +import importlib import random import confidant.clients from confidant import settings +logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) ROLES = [] diff --git a/confidant/services/jwkmanager.py b/confidant/services/jwkmanager.py index 9e7ce1ff..55985cf0 100644 --- a/confidant/services/jwkmanager.py +++ b/confidant/services/jwkmanager.py @@ -1,4 +1,4 @@ -import logging +import importlib from datetime import datetime from datetime import timedelta from datetime import timezone @@ -10,6 +10,7 @@ import jwt from abc import ABC, abstractmethod from cerberus import Validator +from confidant.settings import LOGGING_MODULE from confidant.settings import JWT_ACTIVE_SIGNING_KEYS from confidant.settings import JWT_CACHING_ENABLED from confidant.settings import JWT_CERTIFICATE_AUTHORITIES @@ -24,6 +25,7 @@ from cachetools import TTLCache from jwcrypto import jwk +logging = importlib.import_module(LOGGING_MODULE) logger = logging.getLogger(__name__) CA_SCHEMA = { diff --git a/confidant/services/keymanager.py b/confidant/services/keymanager.py index 92f8b10d..e123c006 100644 --- a/confidant/services/keymanager.py +++ b/confidant/services/keymanager.py @@ -1,5 +1,5 @@ import hashlib -import logging +import importlib import botocore from botocore.exceptions import ClientError @@ -9,6 +9,7 @@ from confidant.utils import stats from confidant.lib import cryptolib +logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) _DATAKEYS = {} diff --git a/confidant/services/webhook.py b/confidant/services/webhook.py index c170148f..6cb38899 100644 --- a/confidant/services/webhook.py +++ b/confidant/services/webhook.py @@ -1,9 +1,10 @@ import requests import json -import logging +import importlib from confidant import settings +logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) diff --git a/confidant/settings.py b/confidant/settings.py index c0b157d0..66a09caa 100644 --- a/confidant/settings.py +++ b/confidant/settings.py @@ -1,12 +1,10 @@ import json -import logging +import importlib from os import getenv from base64 import b64decode from confidant.encrypted_settings import EncryptedSettings -logger = logging.getLogger(__name__) - class SettingsError(Exception): pass @@ -65,6 +63,12 @@ def str_env(var_name, default=''): # Basic setup +# Logging +LOGGING_MODULE = str_env('LOGGING_MODULE', 'logging') + +logging = importlib.import_module(LOGGING_MODULE) +logger = logging.getLogger(__name__) + # Whether or not Confidant is run in debug mode. Never run confidant in debug # mode outside of development! DEBUG = bool_env('DEBUG', False) @@ -691,6 +695,3 @@ def get(name, default=None): # Module that will perform an external ACL check on API endpoints ACL_MODULE = str_env('ACL_MODULE', 'confidant.authnz.rbac:default_acl') - -# Logging -LOGGING_MODULE = str_env('LOGGING_MODULE', 'logging') diff --git a/confidant/utils/maintenance.py b/confidant/utils/maintenance.py index 39b2fa34..d50a3093 100644 --- a/confidant/utils/maintenance.py +++ b/confidant/utils/maintenance.py @@ -1,4 +1,4 @@ -import logging +import importlib import json import os.path from functools import wraps @@ -7,6 +7,7 @@ from confidant import settings +logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) From f7b761047b5a0311ae4e6572df3d7087bb93909f Mon Sep 17 00:00:00 2001 From: Alejandro Roiz Walss Date: Thu, 29 Aug 2024 16:10:32 -0600 Subject: [PATCH 09/13] fix circular dependency --- confidant/encrypted_settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/confidant/encrypted_settings.py b/confidant/encrypted_settings.py index e6b55dee..ce612a8c 100644 --- a/confidant/encrypted_settings.py +++ b/confidant/encrypted_settings.py @@ -6,10 +6,10 @@ from cryptography.fernet import Fernet import confidant.clients -from confidant import settings +from confidant.settings import LOGGING_MODULE from confidant.lib import cryptolib -logging = importlib.import_module(settings.LOGGING_MODULE) +logging = importlib.import_module(LOGGING_MODULE) logger = logging.getLogger(__name__) From 6192941a04a95f565c52bc16fda051a61ca5c53a Mon Sep 17 00:00:00 2001 From: Alejandro Roiz Walss Date: Thu, 29 Aug 2024 16:25:38 -0600 Subject: [PATCH 10/13] fix circular dependency --- confidant/settings.py | 70 +++++++---------------------------------- confidant/utils/misc.py | 52 ++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 58 deletions(-) diff --git a/confidant/settings.py b/confidant/settings.py index 66a09caa..28204241 100644 --- a/confidant/settings.py +++ b/confidant/settings.py @@ -1,74 +1,28 @@ import json import importlib -from os import getenv from base64 import b64decode -from confidant.encrypted_settings import EncryptedSettings - - -class SettingsError(Exception): - pass - - -def bool_env(var_name, default=False): - """ - Get an environment variable coerced to a boolean value. - Example: - Bash: - $ export SOME_VAL=True - settings.py: - SOME_VAL = bool_env('SOME_VAL', False) - Arguments: - var_name: The name of the environment variable. - default: The default to use if `var_name` is not specified in the - environment. - Returns: `var_name` or `default` coerced to a boolean using the following - rules: - "False", "false" or "" => False - Any other non-empty string => True - """ - test_val = getenv(var_name, default) - # Explicitly check for 'False', 'false', and '0' since all non-empty - # string are normally coerced to True. - if test_val in ('False', 'false', '0'): - return False - return bool(test_val) - +from confidant.utils.misc import ( + bool_env, + str_env, + int_env +) -def float_env(var_name, default=0.0): - """ - Get an environment variable coerced to a float value. - This has the same arguments as bool_env. If a value cannot be coerced to a - float, a ValueError will be raised. - """ - return float(getenv(var_name, default)) +# Logging +LOGGING_MODULE = str_env('LOGGING_MODULE', 'logging') +logging = importlib.import_module(LOGGING_MODULE) +logger = logging.getLogger(__name__) -def int_env(var_name, default=0): - """ - Get an environment variable coerced to an integer value. - This has the same arguments as bool_env. If a value cannot be coerced to an - integer, a ValueError will be raised. - """ - return int(getenv(var_name, default)) +from confidant.encrypted_settings import EncryptedSettings -def str_env(var_name, default=''): - """ - Get an environment variable as a string. - This has the same arguments as bool_env. - """ - return getenv(var_name, default) +class SettingsError(Exception): + pass # Basic setup -# Logging -LOGGING_MODULE = str_env('LOGGING_MODULE', 'logging') - -logging = importlib.import_module(LOGGING_MODULE) -logger = logging.getLogger(__name__) - # Whether or not Confidant is run in debug mode. Never run confidant in debug # mode outside of development! DEBUG = bool_env('DEBUG', False) diff --git a/confidant/utils/misc.py b/confidant/utils/misc.py index 3bf320bf..f431cf96 100644 --- a/confidant/utils/misc.py +++ b/confidant/utils/misc.py @@ -1,6 +1,7 @@ import importlib import pytz from datetime import datetime +from os import getenv def dict_deep_update(a, b): @@ -54,3 +55,54 @@ def utcnow(): """ now = datetime.utcnow() return now.replace(tzinfo=pytz.utc) + + +def bool_env(var_name, default=False): + """ + Get an environment variable coerced to a boolean value. + Example: + Bash: + $ export SOME_VAL=True + settings.py: + SOME_VAL = bool_env('SOME_VAL', False) + Arguments: + var_name: The name of the environment variable. + default: The default to use if `var_name` is not specified in the + environment. + Returns: `var_name` or `default` coerced to a boolean using the following + rules: + "False", "false" or "" => False + Any other non-empty string => True + """ + test_val = getenv(var_name, default) + # Explicitly check for 'False', 'false', and '0' since all non-empty + # string are normally coerced to True. + if test_val in ('False', 'false', '0'): + return False + return bool(test_val) + + +def float_env(var_name, default=0.0): + """ + Get an environment variable coerced to a float value. + This has the same arguments as bool_env. If a value cannot be coerced to a + float, a ValueError will be raised. + """ + return float(getenv(var_name, default)) + + +def int_env(var_name, default=0): + """ + Get an environment variable coerced to an integer value. + This has the same arguments as bool_env. If a value cannot be coerced to an + integer, a ValueError will be raised. + """ + return int(getenv(var_name, default)) + + +def str_env(var_name, default=''): + """ + Get an environment variable as a string. + This has the same arguments as bool_env. + """ + return getenv(var_name, default) From 99dd882337f2a39ce35d4486572f4c739bd570f1 Mon Sep 17 00:00:00 2001 From: Alejandro Roiz Walss Date: Thu, 29 Aug 2024 16:32:47 -0600 Subject: [PATCH 11/13] move stats --- confidant/authnz/__init__.py | 2 +- confidant/routes/credentials.py | 3 ++- confidant/routes/jwks.py | 2 +- confidant/routes/services.py | 3 ++- confidant/scripts/restore.py | 2 +- confidant/services/certificatemanager.py | 2 +- confidant/services/credentialmanager.py | 2 +- confidant/services/jwkmanager.py | 2 +- confidant/services/keymanager.py | 2 +- confidant/utils/__init__.py | 9 --------- confidant/utils/stats.py | 9 +++++++++ 11 files changed, 20 insertions(+), 18 deletions(-) create mode 100644 confidant/utils/stats.py diff --git a/confidant/authnz/__init__.py b/confidant/authnz/__init__.py index 0e79bf87..8bef1cba 100644 --- a/confidant/authnz/__init__.py +++ b/confidant/authnz/__init__.py @@ -6,7 +6,7 @@ from functools import wraps from confidant import settings -from confidant.utils import stats +from confidant.utils.stats import stats from confidant.authnz.errors import ( UserUnknownError, diff --git a/confidant/routes/credentials.py b/confidant/routes/credentials.py index c1a4d69c..f1073f0f 100644 --- a/confidant/routes/credentials.py +++ b/confidant/routes/credentials.py @@ -25,7 +25,8 @@ webhook, ) from confidant.services.ciphermanager import CipherManager -from confidant.utils import maintenance, misc, stats +from confidant.utils import maintenance, misc +from confidant.utils.stats import stats from confidant.utils.dynamodb import decode_last_evaluated_key logging = importlib.import_module(settings.LOGGING_MODULE) diff --git a/confidant/routes/jwks.py b/confidant/routes/jwks.py index c0d698ac..51b2b2f0 100644 --- a/confidant/routes/jwks.py +++ b/confidant/routes/jwks.py @@ -2,7 +2,7 @@ from flask import blueprints, jsonify, request -from confidant import authnz +from confidant import authnz, settings from confidant.services.jwkmanager import JWKManager from confidant.schema.jwks import jwt_response_schema, JWTResponse, \ jwks_list_response_schema, JWKSListResponse diff --git a/confidant/routes/services.py b/confidant/routes/services.py index cabd6aaf..bc298a62 100644 --- a/confidant/routes/services.py +++ b/confidant/routes/services.py @@ -20,7 +20,8 @@ servicemanager, webhook, ) -from confidant.utils import maintenance, misc, stats +from confidant.utils import maintenance, misc +from confidant.utils.stats import stats from confidant.utils.dynamodb import decode_last_evaluated_key logging = importlib.import_module(settings.LOGGING_MODULE) diff --git a/confidant/scripts/restore.py b/confidant/scripts/restore.py index e8ce78c9..3d5bb874 100644 --- a/confidant/scripts/restore.py +++ b/confidant/scripts/restore.py @@ -6,7 +6,7 @@ from confidant import settings from confidant.models.credential import Credential, CredentialArchive -from confidant.utils import stats +from confidant.utils.stats import stats logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) diff --git a/confidant/services/certificatemanager.py b/confidant/services/certificatemanager.py index 2cc945dd..d01d1979 100644 --- a/confidant/services/certificatemanager.py +++ b/confidant/services/certificatemanager.py @@ -13,7 +13,7 @@ import confidant.clients from confidant import settings -from confidant.utils import stats +from confidant.utils.stats import stats logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) diff --git a/confidant/services/credentialmanager.py b/confidant/services/credentialmanager.py index a2479ac0..be239bc0 100644 --- a/confidant/services/credentialmanager.py +++ b/confidant/services/credentialmanager.py @@ -6,7 +6,7 @@ from confidant.models.blind_credential import BlindCredential from confidant.models.credential import Credential, CredentialArchive from confidant.models.service import Service -from confidant.utils import stats +from confidant.utils.stats import stats from pynamodb.exceptions import DoesNotExist diff --git a/confidant/services/jwkmanager.py b/confidant/services/jwkmanager.py index 55985cf0..4a282624 100644 --- a/confidant/services/jwkmanager.py +++ b/confidant/services/jwkmanager.py @@ -19,7 +19,7 @@ from confidant.settings import JWT_CACHING_TTL_SECONDS from confidant.settings import REDIS_URL_JWT_CACHE, REDIS_SOCKET_TIMEOUT from confidant.settings import JWT_CACHING_USE_REDIS -from confidant.utils import stats +from confidant.utils.stats import stats from redis import StrictRedis, RedisError from cachetools import TTLCache diff --git a/confidant/services/keymanager.py b/confidant/services/keymanager.py index e123c006..a9857cd0 100644 --- a/confidant/services/keymanager.py +++ b/confidant/services/keymanager.py @@ -6,7 +6,7 @@ import confidant.clients from confidant import settings -from confidant.utils import stats +from confidant.utils.stats import stats from confidant.lib import cryptolib logging = importlib.import_module(settings.LOGGING_MODULE) diff --git a/confidant/utils/__init__.py b/confidant/utils/__init__.py index c144a3f4..e69de29b 100644 --- a/confidant/utils/__init__.py +++ b/confidant/utils/__init__.py @@ -1,9 +0,0 @@ -import statsd - -from confidant import settings - -stats = statsd.StatsClient( - settings.STATSD_HOST, - settings.STATSD_PORT, - prefix='confidant' -) diff --git a/confidant/utils/stats.py b/confidant/utils/stats.py new file mode 100644 index 00000000..c144a3f4 --- /dev/null +++ b/confidant/utils/stats.py @@ -0,0 +1,9 @@ +import statsd + +from confidant import settings + +stats = statsd.StatsClient( + settings.STATSD_HOST, + settings.STATSD_PORT, + prefix='confidant' +) From d3f01176f8067e7b7e5894650f52d87415dbbf88 Mon Sep 17 00:00:00 2001 From: Alejandro Roiz Walss Date: Thu, 29 Aug 2024 16:36:49 -0600 Subject: [PATCH 12/13] fix logging override --- confidant/scripts/archive.py | 5 +++-- confidant/scripts/migrate.py | 5 +++-- confidant/scripts/migrate_bool.py | 5 +++-- confidant/scripts/restore.py | 5 +++-- confidant/scripts/utils.py | 5 +++-- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/confidant/scripts/archive.py b/confidant/scripts/archive.py index 17d23353..6bc9741c 100644 --- a/confidant/scripts/archive.py +++ b/confidant/scripts/archive.py @@ -11,8 +11,9 @@ logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) -logger.addHandler(logging.StreamHandler(sys.stdout)) -logger.setLevel(logging.INFO) +if settings.LOGGING_MODULE == 'logging': + logger.addHandler(logging.StreamHandler(sys.stdout)) + logger.setLevel(logging.INFO) class ArchiveCredentials(Command): diff --git a/confidant/scripts/migrate.py b/confidant/scripts/migrate.py index c32142fe..7cc73cf5 100644 --- a/confidant/scripts/migrate.py +++ b/confidant/scripts/migrate.py @@ -14,8 +14,9 @@ logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) -logger.addHandler(logging.StreamHandler(sys.stdout)) -logger.setLevel(logging.INFO) +if settings.LOGGING_MODULE == 'logging': + logger.addHandler(logging.StreamHandler(sys.stdout)) + logger.setLevel(logging.INFO) def is_old_unicode_set(values): diff --git a/confidant/scripts/migrate_bool.py b/confidant/scripts/migrate_bool.py index f324a3c2..84473a36 100644 --- a/confidant/scripts/migrate_bool.py +++ b/confidant/scripts/migrate_bool.py @@ -16,8 +16,9 @@ logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) -logger.addHandler(logging.StreamHandler(sys.stdout)) -logger.setLevel(logging.INFO) +if settings.LOGGING_MODULE == 'logging': + logger.addHandler(logging.StreamHandler(sys.stdout)) + logger.setLevel(logging.INFO) class GenericCredential(Model): diff --git a/confidant/scripts/restore.py b/confidant/scripts/restore.py index 3d5bb874..9b0ac90a 100644 --- a/confidant/scripts/restore.py +++ b/confidant/scripts/restore.py @@ -11,8 +11,9 @@ logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) -logger.addHandler(logging.StreamHandler(sys.stdout)) -logger.setLevel(logging.INFO) +if settings.LOGGING_MODULE == 'logging': + logger.addHandler(logging.StreamHandler(sys.stdout)) + logger.setLevel(logging.INFO) class RestoreCredentials(Command): diff --git a/confidant/scripts/utils.py b/confidant/scripts/utils.py index cde7ecd4..94b52548 100644 --- a/confidant/scripts/utils.py +++ b/confidant/scripts/utils.py @@ -12,8 +12,9 @@ logging = importlib.import_module(settings.LOGGING_MODULE) logger = logging.getLogger(__name__) -logger.addHandler(logging.StreamHandler(sys.stdout)) -logger.setLevel(logging.INFO) +if settings.LOGGING_MODULE == 'logging': + logger.addHandler(logging.StreamHandler(sys.stdout)) + logger.setLevel(logging.INFO) class ManageGrants(Command): From 165056ecf2dc3962e9aed2a9f4eab6f71bdd5e7c Mon Sep 17 00:00:00 2001 From: Alejandro Roiz Walss Date: Thu, 29 Aug 2024 16:50:22 -0600 Subject: [PATCH 13/13] fix pre-commit --- confidant/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confidant/settings.py b/confidant/settings.py index 28204241..f469e21e 100644 --- a/confidant/settings.py +++ b/confidant/settings.py @@ -14,7 +14,7 @@ logging = importlib.import_module(LOGGING_MODULE) logger = logging.getLogger(__name__) -from confidant.encrypted_settings import EncryptedSettings +from confidant.encrypted_settings import EncryptedSettings # noqa: E402 class SettingsError(Exception):