[archivist] Allow adding a custom username and password

This commit allows defining a username and password separately
from the host. This is necessary because, in some circumstances,
the password may contain special characters that are not compatible
with the default definition of "https://user:password@host:port".

Signed-off-by: Jose Javier Merchante <jjmerchante@bitergia.com>

Author: jjmerchante

src/grimoirelab/core/config/settings.py

@@ -334,6 +334,8 @@
     'WORKERS': int(os.environ.get('GRIMOIRELAB_ARCHIVIST_WORKERS', 10)),
     'STORAGE_TYPE': os.environ.get('GRIMOIRELAB_ARCHIVIST_STORAGE_TYPE', 'opensearch'),
     'STORAGE_URL': os.environ.get('GRIMOIRELAB_ARCHIVIST_STORAGE_URL', 'https://admin:admin@localhost:9200'),
+    'STORAGE_USERNAME': os.environ.get('GRIMOIRELAB_ARCHIVIST_STORAGE_USERNAME', ''),
+    'STORAGE_PASSWORD': os.environ.get('GRIMOIRELAB_ARCHIVIST_STORAGE_PASSWORD', ''),
     'STORAGE_INDEX': os.environ.get('GRIMOIRELAB_ARCHIVIST_STORAGE_INDEX', 'events'),
     'STORAGE_VERIFY_CERT': os.environ.get('GRIMOIRELAB_ARCHIVIST_STORAGE_VERIFY_CERT', 'False').lower() in ('true', '1'),
     'EVENTS_PER_JOB': int(os.environ.get('GRIMOIRELAB_ARCHIVIST_EVENTS_PER_JOB', 10000)),

src/grimoirelab/core/runner/commands/run.py

@@ -155,6 +155,8 @@ def create_background_tasks(clear_tasks: bool):
 
     workers = settings.GRIMOIRELAB_ARCHIVIST['WORKERS']
     storage_url = settings.GRIMOIRELAB_ARCHIVIST['STORAGE_URL']
+    storage_username = settings.GRIMOIRELAB_ARCHIVIST['STORAGE_USERNAME']
+    storage_password = settings.GRIMOIRELAB_ARCHIVIST['STORAGE_PASSWORD']
     storage_db_name = settings.GRIMOIRELAB_ARCHIVIST['STORAGE_INDEX']
     storage_type = settings.GRIMOIRELAB_ARCHIVIST['STORAGE_TYPE']
     verify_certs = settings.GRIMOIRELAB_ARCHIVIST['STORAGE_VERIFY_CERT']
@@ -172,6 +174,8 @@ def create_background_tasks(clear_tasks: bool):
 
     task_args = {
         'storage_url': storage_url,
+        'storage_username': storage_username,
+        'storage_password': storage_password,
         'storage_db_name': storage_db_name,
         'storage_verify_certs': verify_certs,
         'redis_group': 'archivist',

src/grimoirelab/core/scheduler/tasks/archivist.py

@@ -41,6 +41,8 @@
 def archivist_job(
     storage_type: str,
     storage_url: str,
+    storage_username: str,
+    storage_password: str,
     storage_db_name: str,
     storage_verify_certs: bool,
     redis_group: str,
@@ -56,6 +58,8 @@ def archivist_job(
 
     :param storage_type: type of the storage system (e.g., 'opensearch')
     :param storage_url: URL of the storage system
+    :param storage_username: Username to use when authentication is required
+    :param storage_password: Password to use when authentication is required
     :param storage_db_name: Name of the database to use
     :param storage_verify_certs: Verify certificates when connecting to the storage system
     :param redis_group: Redis group name to use for fetching events
@@ -77,6 +81,8 @@ def archivist_job(
 
     Storage = get_storage_backend(storage_type)
     storage = Storage(url=storage_url,
+                      user=storage_username,
+                      password=storage_password,
                       db_name=storage_db_name,
                       verify_certs=storage_verify_certs)
     events = events_consumer(rq_job.connection,
@@ -272,8 +278,17 @@ class StorageBackend:
 
     :param url: URL of the storage backend
     """
-    def __init__(self, url: str, db_name: str, verify_certs: bool = False) -> None:
+    def __init__(
+        self,
+        url: str,
+        db_name: str,
+        user: str | None = None,
+        password: str | None = None,
+        verify_certs: bool = False
+    ) -> None:
         self.url = url
+        self.user = user
+        self.password = password
         self.db_name = db_name
         self.verify_certs = verify_certs
 
@@ -361,13 +376,24 @@ class OpenSearchStorage(StorageBackend):
         }
     }
 
-    def __init__(self, url: str, db_name: str, verify_certs: bool = False) -> None:
-        super().__init__(url, db_name, verify_certs)
+    def __init__(
+            self,
+            url: str,
+            db_name: str,
+            user: str | None = None,
+            password: str | None = None,
+            verify_certs: bool = False
+    ) -> None:
+        super().__init__(url=url, db_name=db_name, user=user, password=password, verify_certs=verify_certs)
 
         if not verify_certs:
             urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
-        self.client = OpenSearch([url], verify_certs=self.verify_certs)
+        auth = None
+        if user and password:
+            auth = (user, password)
+
+        self.client = OpenSearch([url], http_auth=auth, verify_certs=self.verify_certs)
         self._create_index(db_name)
         self.max_items_bulk = 100
 

src/grimoirelab/core/scheduler/tasks/models.py

@@ -231,6 +231,8 @@ def prepare_job_parameters(self):
         task_args = {
             'storage_type': self.storage_type,
             'storage_url': self.task_args.get('storage_url'),
+            'storage_username': self.task_args.get('storage_username'),
+            'storage_password': self.task_args.get('storage_password'),
             'storage_db_name': self.task_args.get('storage_db_name'),
             'storage_verify_certs': self.task_args.get('storage_verify_certs'),
             'redis_group': self.task_args.get('redis_group'),

tests/scheduler/test_task_archivist.py

@@ -88,6 +88,8 @@ def test_job(self):
         job_args = {
             'storage_type': 'mock_storage',
             'storage_url': 'example.com',
+            'storage_username': 'user',
+            'storage_password': 'password',
             'storage_db_name': 'mock_db',
             'storage_verify_certs': True,
             'redis_group': 'archivist',
@@ -127,6 +129,8 @@ def test_job_no_result(self):
         job_args = {
             'storage_type': 'mock_storage',
             'storage_url': 'example.com',
+            'storage_username': 'user',
+            'storage_password': 'password',
             'storage_db_name': 'mock_db',
             'storage_verify_certs': True,
             'redis_group': 'archivist',
@@ -167,6 +171,8 @@ def test_backend_not_found(self):
         job_args = {
             'storage_type': 'nobackend',
             'storage_url': 'example.com',
+            'storage_username': 'user',
+            'storage_password': 'password',
             'storage_db_name': 'mock_db',
             'storage_verify_certs': True,
             'redis_group': 'archivist',