Atlas http endpoint error handling and probe metric added (#227)

* atlas http endpoint probe metric added * simplejson added * atlas endpoint probe only if used; switch to exit code 1 * Test also expects exit code 1 * take suggestion Co-authored-by: Tommy Sauer <[email protected]> * take this suggestion too Co-authored-by: Tommy Sauer <[email protected]> Co-authored-by: Tommy Sauer <[email protected]>
sapcc · Jun 21, 2022 · 9bae931 · 9bae931
1 parent 4d162e8
commit 9bae931
Show file tree

Hide file tree

Showing 6 changed files with 46 additions and 16 deletions.
diff --git a/InventoryBuilder.py b/InventoryBuilder.py
@@ -7,7 +7,9 @@
 from collections import defaultdict
 import time
 import json
+import simplejson
 import os
+import sys
 import logging
 import requests
 
@@ -172,13 +174,29 @@ def get_vrops(self):
         elif self.atlas_path:
             try:
                 response = requests.get(url=self.atlas_path)
-                netbox_json = response.json()
-                self.vrops_list = [target['labels']['server_name'] for target in netbox_json if
-                                   target['labels']['job'] == "vrops"]
+                self.response_codes['atlas'][self.atlas_path] = response.status_code
             except (urllib3.exceptions.MaxRetryError, requests.exceptions.ConnectionError,
-                    urllib3.exceptions.NewConnectionError):
-                logger.error(f'Failed to establish a connection to: {self.atlas_path}, retrying in {self.sleep}s')
-                logger.info(f'continue with the old vrops_list')
+                    urllib3.exceptions.NewConnectionError) as err:
+                logger.error(f'Failed to establish a connection to: {self.atlas_path}, retrying in {self.sleep}s', err)
+                if not self.vrops_list:
+                    logger.critical(f'No targets found to start InventoryBuilder, bailing out')
+                    sys.exit(1)
+                logger.info(f'Continue with the old vrops_list')
+                return
+
+            try:
+                response.json()
+            except simplejson.errors.JSONDecodeError:
+                logger.error(f'Invalid json data in atlas netbox http response, Content-Type: "{response.headers.get("Content-Type")}"')
+                if not self.vrops_list:
+                    logger.critical(f'No targets found to start InventoryBuilder, bailing out')
+                    sys.exit(1)
+                logger.info(f'Continue with the old vrops_list')
+                return
+
+            netbox_json = response.json()
+            self.vrops_list = [target['labels']['server_name'] for target in netbox_json if target['labels']['job'] == "vrops"]
+
         else:
             return
 

diff --git a/collectors/InventoryCollector.py b/collectors/InventoryCollector.py
@@ -35,6 +35,9 @@ def collect(self):
             yield self.collection_time_metric(target)
             yield self.inventory_targets_info(target)
 
+            # If Atlas is used for target discovery
+            yield self.atlas_http_sd_endpoint_probe()
+
     def amount_inventory_resources(self, target):
         gauges = list()
         for resourcekind, amount in self.get_amount_resources()[target].items():
@@ -63,6 +66,14 @@ def api_response_metric(self, target):
                                           value=status_code)
         return api_response_gauge
 
+    def atlas_http_sd_endpoint_probe(self):
+        atlas_response_gauge = GaugeMetricFamily('atlas_sd_response', 'vrops_inventory',
+                                                 labels=['atlas_path'])
+        if atlas_endpoint_response := self.get_inventory_api_responses().get('atlas'):
+            for atlas_path, response_code in atlas_endpoint_response.items():
+                atlas_response_gauge.add_metric(labels=[atlas_path], value=response_code)
+        return atlas_response_gauge
+
     def collection_time_metric(self, target):
         collection_time_gauge = GaugeMetricFamily('vrops_inventory_collection_time_seconds', 'vrops_inventory',
                                                   labels=["target"])

diff --git a/exporter.py b/exporter.py
@@ -68,16 +68,16 @@ def parse_params(logger):
 
     if "PORT" not in os.environ and not options.port:
         logger.error('Cannot start, please specify port with ENV or -o')
-        sys.exit(0)
+        sys.exit(1)
     if "INVENTORY" not in os.environ and not options.inventory:
         logger.error('Cannot start, please specify inventory with ENV or -i')
-        sys.exit(0)
+        sys.exit(1)
     if "COLLECTOR_CONFIG" not in os.environ and not options.config:
         logger.error('Cannot start, please specify collector config with ENV or -m')
-        sys.exit(0)
+        sys.exit(1)
     if not options.collectors:
         logger.error('Cannot start, no default collectors activated in config')
-        sys.exit(0)
+        sys.exit(1)
 
     return options
 

diff --git a/inventory.py b/inventory.py
@@ -70,21 +70,21 @@ def parse_params(logger):
 
     if "PORT" not in os.environ and not options.port:
         logger.error('Cannot start, please specify PORT with ENV or -o')
-        sys.exit(0)
+        sys.exit(1)
     if "USER" not in os.environ and not options.user:
         logger.error('Cannot start, please specify USER with ENV or -u')
-        sys.exit(0)
+        sys.exit(1)
     if "PASSWORD" not in os.environ and not options.password:
         logger.error('Cannot start, please specify PASSWORD with ENV or -p')
-        sys.exit(0)
+        sys.exit(1)
     if "INVENTORY_CONFIG" not in os.environ and not options.config:
         logger.error('Cannot start, please specify inventory config with ENV or -m')
-        sys.exit(0)
+        sys.exit(1)
     if "ATLAS" not in os.environ and not options.atlas:
         vrops_list = yaml_read(os.environ['INVENTORY_CONFIG']).get('vrops_targets')
         if not vrops_list:
             logger.error('Cannot start, please declare vrops_targets in inventory config or ATLAS path with ENV or -a')
-            sys.exit(0)
+            sys.exit(1)
 
     return options
 

diff --git a/requirements.txt b/requirements.txt
@@ -11,3 +11,4 @@ beautifulsoup4==4.9.3
 lxml
 itsdangerous==1.1.0
 jinja2==3.0.3
+simplejson
diff --git a/tests/TestLaunchInventory.py b/tests/TestLaunchInventory.py
@@ -108,7 +108,7 @@ def test_without_params(self):
         sys.argv = ['prog']
         with self.assertRaises(SystemExit) as se:
             parse_params(logger)
-        self.assertEqual(se.exception.code, 0, 'PORT, USER, ATLAS or PASSWORD are not set in ENV or command line!')
+        self.assertEqual(se.exception.code, 1, 'PORT, USER, ATLAS or PASSWORD are not set in ENV or command line!')
 
 
 if __name__ == '__main__':