Merge pull request #248 from fact-project/shifthelper_heartbeat

Heartbeats

Author: maxnoe

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name='shifthelper',
-    version='1.1.4',
+    version='1.2.0',
     description='a tool for helping people with a FACT night shift',
     url='https://github.com/fact-project/shifthelper',
     author='Dominik Neise, Maximilian Noethe, Sebastian Mueller',

shifthelper/__main__.py

@@ -9,7 +9,7 @@
 from .tools.shift import get_current_shifter
 from .tools import config
 from .logging import config_logging
-from .checks import FactIntervalCheck, FlareAlertCheck
+from .checks import FactIntervalCheck
 from . import conditions
 from .categories import CATEGORY_SHIFTER, CATEGORY_DEVELOPER
 
@@ -54,7 +54,14 @@ def telegram_book(category):
 def main():
     with Custos(
             checks=[
-                FlareAlertCheck(category=CATEGORY_SHIFTER, interval=300),
+                FactIntervalCheck(
+                    name='HeartBeat',
+                    interval=120,
+                    checklist=[
+                        conditions.update_heartbeat,
+                    ],
+                    category=CATEGORY_DEVELOPER
+                ),
                 FactIntervalCheck(
                     name='DummyAlert',
                     interval=60,

shifthelper/conditions.py

@@ -17,6 +17,7 @@
 from .tools import get_last_parking_checklist_entry
 from .tools import fetch_users_awake
 from .tools import fetch_dummy_alerts
+from . import tools
 from . import retry_smart_fact_crawler as sfc
 from .debug_log_wrapper import log_call_and_result
 
@@ -277,6 +278,24 @@ def is_nobody_awake():
         return get_current_shifter().username not in awake
 
 
+@log_call_and_result
+def update_heartbeat():
+    '''HeartbeatMonitor not ok'''
+    log = logging.getLogger(__name__)
+    heartbeats = tools.update_heartbeat()
+    if "heartbeatMonitor" not in heartbeats:
+        log.debug("HeartbeatMonitor offline?")
+        return True
+    else:
+        heartbeat_monitor_age = (
+            datetime.utcnow() -
+            pd.to_datetime(heartbeats['heartbeatMonitor'])
+        )
+        if heartbeat_monitor_age > timedelta(minutes=10):
+            log.debug('heartbeat_monitor_age > timedelta(minutes=10)')
+            return True
+    return False
+
 @log_call_and_result
 def is_dummy_alert_by_shifter():
     '''Dummy Alert'''

shifthelper/notifiers.py

@@ -1,9 +1,7 @@
 import datetime
-import requests
 
 from custos import TwilioNotifier
 from .tools.shift import get_current_shifter
-from copy import copy
 from .tools import config, get_alerts
 from .categories import CATEGORY_DEVELOPER
 
@@ -37,32 +35,7 @@ def notify(self, recipient, msg):
             )
         self.not_acknowledged_messages.append(msg)
 
-    def _remove_acknowledged_and_old_calls(self):
-        """ from the list of not acknowledged calls
-        remove all calls, which have been acknowledged on the web page
-
-        Also remove calls older than 2 hours, to get out of
-        a "call the backup shifter" dead lock
-        """
-        try:
-            alerts = {a['uuid']: a for a in get_alerts()}
-        except requests.exceptions.RequestException:
-            return
-
-        for msg in copy(self.not_acknowledged_messages):
-            age = datetime.datetime.utcnow() - msg.timestamp
-            if age > (self.max_time_for_fallback + self.time_before_fallback):
-                self.not_acknowledged_messages.remove(msg)
-            else:
-                try:
-                    alert = alerts[str(msg.uuid)]
-                except KeyError:
-                    continue
-
-                if alert['acknowledged'] is True:
-                    self.not_acknowledged_messages.remove(msg)
-
-    def _get_oldest_call_age(self):
+    def _get_oldest_message_age(self):
         max_age = datetime.timedelta()
         for msg in self.not_acknowledged_messages:
             age = datetime.datetime.utcnow() - msg.timestamp
@@ -97,14 +70,16 @@ def get_numbers_to_call(self, msg):
             log.debug('Getting phone number of primary shifter')
             numbers_to_call.append(self.phone_number_of_normal_shifter())
 
-            if self._get_oldest_call_age() >= self.time_before_fallback:
+            if self._get_oldest_message_age() >= self.time_before_fallback:
                 log.debug('Getting phone number of fallback shifter')
                 numbers_to_call.append(self.phone_number_of_fallback_shifter())
 
         return numbers_to_call
 
     def handle_message(self, msg):
-        self._remove_acknowledged_and_old_calls()
+        self._remove_old_messages()
+        self._remove_acknowledged_messages()
+
         log.debug('Got a message')
         if msg.level >= self.level:
             log.debug('Message is over alert level')
@@ -113,3 +88,37 @@ def handle_message(self, msg):
             for phone_number in numbers_to_call:
                 log.info('Calling {}'.format(phone_number))
                 self.notify(phone_number, msg)
+
+    def _remove_old_messages(self):
+        """ from the list of not_acknowledged_messages
+        remove messages older than a certain limit, to avoid calling the
+        fallback forever.
+        """
+        limit = self.max_time_for_fallback + self.time_before_fallback
+        self.not_acknowledged_messages = [
+            msg for msg in self.not_acknowledged_messages
+            if not is_message_old(msg, limit)
+        ]
+
+    def _remove_acknowledged_messages(self):
+        ''' from the list of not_acknowledged_messages
+        remove all messages, which have been acknowledged on the web page
+        '''
+        alerts = {a['uuid']: a for a in get_alerts()}
+
+        self.not_acknowledged_messages = [
+            msg for msg in self.not_acknowledged_messages
+            if not is_message_acknowledged(alerts, msg)
+        ]
+
+
+def is_message_acknowledged(alerts, msg):
+    try:
+        return alerts[str(msg.uuid)]['acknowledged']
+    except KeyError:
+        return False
+
+
+def is_message_old(msg, limit):
+    age = datetime.datetime.utcnow() - msg.timestamp
+    return age > limit

shifthelper/tools/__init__.py

@@ -26,13 +26,19 @@
 db_engines = {}
 
 
-@retry(stop_max_delay=30000,  # 30 seconds max
-       wait_exponential_multiplier=100,  # wait 2^i * 100 ms, on the i-th retry
-       wait_exponential_max=1000,  # but wait 1 second per try maximum
-       )
 def get_alerts():
-    alerts = requests.get(config['webservice']['post-url'])
-    return alerts.json()
+    @retry(stop_max_delay=30000,  # 30 seconds max
+           wait_exponential_multiplier=100,  # wait 2^i * 100 ms, on the i-th retry
+           wait_exponential_max=1000,  # but wait 1 second per try maximum
+           wrap_exception=True
+           )
+    def retry_fetch_fail_after_30sec():
+        alerts = requests.get(config['webservice']['post-url'])
+        return alerts.json()
+    try:
+        return retry_fetch_fail_after_30sec()
+    except RetryError:
+        return {}
 
 
 def create_db_connection(db_config=None):
@@ -103,6 +109,27 @@ def retry_fetch_fail_after_30sec():
         return {}
 
 
+def update_heartbeat():
+    @retry(
+        stop_max_delay=30000,  # 30 seconds max
+        wait_exponential_multiplier=100,  # wait 2^i * 100 ms, on the i-th retry
+        wait_exponential_max=1000,  # but wait 1 second per try maximum
+        wrap_exception=True
+    )
+    def retry_fetch_fail_after_30sec():
+        return requests.post(
+            config['webservice']['shifthelperHeartbeat'],
+            auth=(
+                config['webservice']['user'],
+                config['webservice']['password']
+            )
+        ).json()
+    try:
+        return retry_fetch_fail_after_30sec()
+    except RetryError as e:
+        return {}
+
+
 class NightlyResettingDefaultdict(defaultdict):
     def __init__(self, *args, **kwargs):
         self.night = night_integer(datetime.utcnow())

tests/test_acknowledging.py

@@ -32,7 +32,7 @@ def test_all_acknowledged():
     )
 
 
-def test_message_level_all_acknowledged():
+def test_message_level_all_acknowledged_1():
     from shifthelper.checks import message_level
 
     alerts = pd.read_json('tests/resources/not_all_acknowledged.json')
@@ -44,7 +44,7 @@ def test_message_level_all_acknowledged():
     assert message_level(checkname='MainJSStatusCheck', alerts=alerts) == levels.WARNING
 
 
-def test_message_level_not_all_acknowledged():
+def test_message_level_not_all_acknowledged_2():
     from shifthelper.checks import message_level
 
     alerts = pd.read_json('tests/resources/not_all_acknowledged.json')

tests/test_conditions.py

@@ -1,5 +1,4 @@
 from freezegun import freeze_time
-import datetime
 import smart_fact_crawler