systemd-watchdog: improve notifications to watchdog (#595)

system will now be notified on send timeouts
and when event handling is running for a long time

Signed-off-by: Alexander Mohr <[email protected]>

Author: alexmohr

src/daemon/dlt-daemon.c

@@ -95,10 +95,6 @@ static int dlt_daemon_check_numeric_setting(char *token,
                                             char *value,
                                             unsigned long *data);
 
-#ifdef DLT_SYSTEMD_WATCHDOG_ENABLE
-static uint32_t watchdog_trigger_interval;  /* watchdog trigger interval in [s] */
-#endif
-
 /* used in main event loop and signal handler */
 int g_exit = 0;
 
@@ -1268,7 +1264,7 @@ int main(int argc, char *argv[])
         if (watchdogUSec)
             watchdogTimeoutSeconds = atoi(watchdogUSec) / 2000000;
 
-        watchdog_trigger_interval = watchdogTimeoutSeconds;
+        daemon.watchdog_trigger_interval = watchdogTimeoutSeconds;
         create_timer_fd(&daemon_local,
                         watchdogTimeoutSeconds,
                         watchdogTimeoutSeconds,
@@ -3756,7 +3752,7 @@ int dlt_daemon_send_ringbuffer_to_client(DltDaemon *daemon, DltDaemonLocal *daem
     static uint8_t data[DLT_DAEMON_RCVBUFSIZE];
     int length;
 #ifdef DLT_SYSTEMD_WATCHDOG_ENABLE
-    uint32_t curr_time;
+    uint32_t curr_time = 0U;
 #endif
 
     PRINT_FUNCTION_VERBOSE(verbose);
@@ -3771,24 +3767,9 @@ int dlt_daemon_send_ringbuffer_to_client(DltDaemon *daemon, DltDaemonLocal *daem
         return DLT_DAEMON_ERROR_OK;
     }
 
-#ifdef DLT_SYSTEMD_WATCHDOG_ENABLE
-
-    if (sd_notify(0, "WATCHDOG=1") < 0)
-        dlt_vlog(LOG_WARNING, "Could not reset systemd watchdog: %s\n", strerror(errno));
-
-    curr_time = dlt_uptime();
-#endif
-
     while ((length = dlt_buffer_copy(&(daemon->client_ringbuffer), data, sizeof(data))) > 0) {
 #ifdef DLT_SYSTEMD_WATCHDOG_ENABLE
-
-        if ((dlt_uptime() - curr_time) / 10000 >= watchdog_trigger_interval) {
-            if (sd_notify(0, "WATCHDOG=1") < 0)
-                dlt_log(LOG_WARNING, "Could not reset systemd watchdog\n");
-
-            curr_time = dlt_uptime();
-        }
-
+        dlt_daemon_trigger_systemd_watchdog_if_necessary(&curr_time, daemon->watchdog_trigger_interval);
 #endif
 
         if ((ret =

src/daemon/dlt_daemon_common.c

@@ -86,6 +86,10 @@
 #include "dlt_daemon_socket.h"
 #include "dlt_daemon_serial.h"
 
+#ifdef DLT_SYSTEMD_WATCHDOG_ENABLE
+#   include <systemd/sd-daemon.h>
+#endif
+
 char *app_recv_buffer = NULL; /* pointer to receiver buffer for application msges */
 
 static int dlt_daemon_cmp_apid(const void *m1, const void *m2)
@@ -1843,3 +1847,14 @@ void dlt_daemon_change_state(DltDaemon *daemon, DltDaemonState newState)
         break;
     }
 }
+
+#ifdef DLT_SYSTEMD_WATCHDOG_ENABLE
+void dlt_daemon_trigger_systemd_watchdog_if_necessary(unsigned int* last_trigger_time, unsigned int watchdog_trigger_interval) {
+    unsigned int uptime = dlt_uptime();
+    if ((uptime - *last_trigger_time) / 10000 >= watchdog_trigger_interval) {
+        if (sd_notify(0, "WATCHDOG=1") < 0)
+            dlt_vlog(LOG_WARNING, "%s: Could not reset systemd watchdog\n", __func__);
+        *last_trigger_time = uptime;
+    }
+}
+#endif

src/daemon/dlt_daemon_common.h

@@ -200,6 +200,9 @@ typedef struct
 #ifdef DLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE
     int received_message_since_last_watchdog_interval;
 #endif
+#ifdef DLT_SYSTEMD_WATCHDOG_ENABLE
+    unsigned int watchdog_trigger_interval;  /* watchdog trigger interval in [s] */
+#endif
 #ifdef DLT_LOG_LEVEL_APP_CONFIG
     DltDaemonContextLogSettings *app_id_log_level_settings; /**< Settings for app id specific log levels */
     int num_app_id_log_level_settings;  /** < count of log level settings */
@@ -563,6 +566,10 @@ void dlt_daemon_control_reset_to_factory_default(DltDaemon *daemon,
  */
 void dlt_daemon_change_state(DltDaemon *daemon, DltDaemonState newState);
 
+#ifdef DLT_SYSTEMD_WATCHDOG_ENABLE
+void dlt_daemon_trigger_systemd_watchdog_if_necessary(unsigned int* last_trigger_time, unsigned int watchdog_trigger_interval);
+#endif
+
 #   ifdef __cplusplus
 }
 #   endif

src/daemon/dlt_daemon_event_handler.c

@@ -42,6 +42,7 @@
 #include "dlt_daemon_connection_types.h"
 #include "dlt_daemon_event_handler.h"
 #include "dlt_daemon_event_handler_types.h"
+#include "dlt_daemon_common.h"
 
 /**
  * \def DLT_EV_TIMEOUT_MSEC
@@ -205,6 +206,10 @@ int dlt_daemon_handle_event(DltEventHandler *pEvent,
         return ret;
     }
 
+#ifdef DLT_SYSTEMD_WATCHDOG_ENABLE
+    unsigned int start_time = dlt_uptime();
+#endif
+
     for (i = 0; i < pEvent->nfds; i++) {
         int fd = 0;
         DltConnection *con = NULL;
@@ -248,7 +253,8 @@ int dlt_daemon_handle_event(DltEventHandler *pEvent,
         if (!callback) {
             dlt_vlog(LOG_CRIT, "Unable to find function for %u handle type.\n",
                      type);
-            return -1;
+            /* keep handling remaining events */
+            continue;
         }
 
         /* From now on, callback is correct */
@@ -260,6 +266,9 @@ int dlt_daemon_handle_event(DltEventHandler *pEvent,
                      type);
             return -1;
         }
+#ifdef DLT_SYSTEMD_WATCHDOG_ENABLE
+        dlt_daemon_trigger_systemd_watchdog_if_necessary(&start_time, daemon->watchdog_trigger_interval);
+#endif
     }
 
     return 0;

src/daemon/dlt_daemon_socket.c

@@ -48,6 +48,10 @@
 #include <linux/stat.h>
 #endif
 
+#ifdef DLT_SYSTEMD_WATCHDOG_ENABLE
+#include <systemd/sd-daemon.h>
+#endif
+
 #include "dlt_types.h"
 #include "dlt-daemon.h"
 #include "dlt-daemon_cfg.h"
@@ -224,6 +228,14 @@ int dlt_daemon_socket_sendreliable(int sock, void *data_buffer, int message_size
         if (ret < 0) {
             dlt_vlog(LOG_WARNING,
                      "%s: socket send failed [errno: %d]!\n", __func__, errno);
+#ifdef DLT_SYSTEMD_WATCHDOG_ENABLE
+            /* notify systemd here that we are still alive
+             * otherwise we might miss notifying the watchdog when
+             * the watchdog interval is small and multiple timeouts occur back to back
+             */
+            if (sd_notify(0, "WATCHDOG=1") < 0)
+                dlt_vlog(LOG_WARNING, "%s: Could not reset systemd watchdog\n", __func__);
+#endif
             return DLT_DAEMON_ERROR_SEND_FAILED;
         } else {
             data_sent += ret;

tests/CMakeLists.txt

@@ -19,9 +19,9 @@ else()
     set(LIBRARIES socket)
 endif()
 
-if(WITH_SYSTEMD)
-  set(SYSTEMD_LIBS systemd)
-endif(WITH_SYSTEMD)
+if (WITH_SYSTEMD OR WITH_SYSTEMD_SOCKET_ACTIVATION OR WITH_SYSTEMD_WATCHDOG)
+    set(LIBRARIES ${LIBRARIES} systemd)
+endif()
 
 set(DLT_LIBRARIES dlt ${GTEST_LIBS} ${LIBRARIES})
 set(DLT_DAEMON_LIBRARIES dlt_daemon ${GTEST_LIBS} ${LIBRARIES} ${SYSTEMD_LIBS})