First skeleton for the wsrep node example

Author: ayurchen

.gitignore

@@ -1,3 +1,4 @@
+*~
 *.o
 *.a
 *.diff
@@ -6,3 +7,4 @@ CMakeFiles
 CMakeCache.txt
 cmake_install.cmake
 Makefile
+examples/node/node

CMakeLists.txt

@@ -19,6 +19,10 @@ INCLUDE_DIRECTORIES(".")
 
 SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Werror -Wconversion")
 
+IF (NOT CMAKE_BUILD_TYPE)
+    SET(CMAKE_BUILD_TYPE Release)
+ENDIF()
+
 SET(WSREP_SOURCES wsrep_gtid.c wsrep_uuid.c wsrep_loader.c wsrep_dummy.c)
 
 ADD_LIBRARY(wsrep ${WSREP_SOURCES})

README.md

@@ -2,5 +2,6 @@
 
 Building:
 ```
-cmake . && make
+cmake [-DCMAKE_BUILD_TYPE=Debug|Release] . && make [VERBOSE=1]
 ```
+in top directory.

examples/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2018, Codership Oy. All rights reserved.
+# Copyright (c) 2019, Codership Oy. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,3 +15,5 @@
 
 ADD_EXECUTABLE(listener listener.c)
 TARGET_LINK_LIBRARIES(listener wsrep dl pthread)
+
+ADD_SUBDIRECTORY(node)

examples/README.md

@@ -3,3 +3,7 @@
 ### 1. Listener
 Is a simple program that connects and listens to replication events in
 an existing cluster.
+
+### 2. Node
+Is a more complex program which implements most of wsrep node functionality
+and can form clusters in itself.

examples/node/CMakeLists.txt

@@ -0,0 +1,23 @@
+# Copyright (c) 2019, Codership Oy. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+
+FILE(GLOB SRC
+  "*.h"
+  "*.c"
+  )
+
+ADD_EXECUTABLE(node ${SRC})
+
+TARGET_LINK_LIBRARIES(node wsrep dl pthread)

examples/node/README.md

@@ -0,0 +1,77 @@
+# wsrep API node application
+
+## Overview
+
+This is a simple application to demonstrate the usage of wsrep API. It
+deliberately does nothing useful in order to present as concentrated and
+concise API usage as possible.
+
+The program is deliberately written in C to demonstrate the naked API usage.
+For C++ example see a much more advanced integration library at
+https://github.com/codership/wsrep-lib
+
+## High level architecture
+
+Process-wise the program consists of an endless main loop that periodically
+samples and prints performance stats and a configurable number of "master" and
+"slave" threads, with master threads loop processing "transactions" and
+replicating resulting "write sets" and slave threads receiving and processing
+the write sets from other nodes.
+
+Object-wise the program is composed of two main objects: `store` and `wsrep`.
+'store' object contains application "state" and generates and commits changes
+to the state. `wsrep` object contains cluster context and provides interface
+to it. Changes generated by `store` are replicated and certified through
+`wsrep` and then committed to `store`.
+
+## Unit descriptions (in alphabetical order)
+
+#### ctx.h
+A small header to declare the application context structure.
+
+#### log.*
+Implements logging functionality for the application AND
+**a logging callback** for the wsrep provider.
+
+#### main.c
+Defines `main()` routine that initializes storage and wsrep provider, starts
+the worker threads and loops in a statistics collection loop. Even though it is
+not designed to return it still shows the deinitialization order.
+
+#### options.*
+Implements reading configuration options from the command line, does not have
+anything related to wsrep API, but shows which additional parameters must be
+configured for the program to make use of wsrep clustering.
+
+#### sst.*
+Defines **SST callbacks** for the wsrep provider and shows how to asynchronously
+implement state snapshot transfer (yes, you don't want to spend eternity in
+callbacks).
+
+#### stats.*
+Implements performance stats collecting function for the main loop. While it is
+an absolutely optional provider functionality, still it shows how to use that.
+
+#### store.*
+Defines the `store` object that pretends to store and modify some data in a
+"transactional" manner. It provides the caller that intends to do a change with
+a *change data* and a *key* for replication and certification.
+
+#### trx.*
+Defines routines to process local and replicated transactions.
+
+#### worker.*
+Implements worker thread pool functinality. Worker threads run routines defined
+in 'trx.*'. Also implements **apply callback** for the wsrep provider.
+
+#### wsrep.*
+Maintains wsrep cluster context: provider instance and cluster membership view.
+While there is little use for the latter in this primitive application, still
+it shows **connected and view callbacks** usage. But mostly, for this
+application its purpose is to initialize the provider, connect to the cluster
+and offer access to initialized provider for other parts of the program.
+
+## Example usage
+```
+./node -f /tmp/galera/0 -v /tmp/galera/0/galera/lib/libgalera_smm.so -o 'pc.weight=2;evs.send_window=2;evs.user_send_window=1;gcache.recover=no' -s 8 -m 16
+```

examples/node/ctx.h

@@ -0,0 +1,34 @@
+/* Copyright (c) 2019, Codership Oy. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ */
+
+/**
+ * @file This unit defines application context for wsrep provider
+ */
+
+#ifndef NODE_CTX_H
+#define NODE_CTX_H
+
+#include "store.h"
+#include "wsrep.h"
+
+struct node_ctx
+{
+    struct node_wsrep*         wsrep;
+    struct node_store*         store;
+    const struct node_options* opts;
+};
+
+#endif /* NODE_CTX_H */

examples/node/log.c

@@ -0,0 +1,100 @@
+/* Copyright (c) 2019, Codership Oy. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ */
+
+#include "log.h"
+
+#include <stdio.h>    // fprintf(), fflush()
+#include <sys/time.h> // gettimeofday()
+#include <time.h>     // localtime_r()
+#include <stdarg.h>   // va_start(), va_end()
+
+wsrep_log_level_t node_log_max_level = WSREP_LOG_INFO;
+
+static const char* log_level_str[WSREP_LOG_DEBUG + 2] =
+{
+    "FATAL: ",
+    "ERROR: ",
+    " WARN: ",
+    " INFO: ",
+    "DEBUG: ",
+    "XXXXX: "
+};
+
+static inline void
+log_timestamp_and_log(const char* const prefix, // source of msg
+                      int         const severity,
+                      const char* const msg)
+{
+    struct tm      date;
+    struct timeval time;
+
+    gettimeofday(&time, NULL);
+    localtime_r (&time.tv_sec, &date);
+
+    FILE* log_file = stderr;
+    fprintf(log_file,
+            "%04d-%02d-%02d %02d:%02d:%02d.%03d " /* timestamp fmt */
+            "[%s] %s%s\n",                        /* [prefix] severity msg */
+            date.tm_year + 1900, date.tm_mon + 1, date.tm_mday,
+            date.tm_hour, date.tm_min, date.tm_sec,
+            (int)time.tv_usec / 1000,
+            prefix, log_level_str[severity], msg
+        );
+
+    fflush (log_file);
+}
+
+void
+node_log_cb(wsrep_log_level_t const severity, const char* const msg)
+{
+    /* REPLICATION: let provider log messages be prefixed with 'wsrep'*/
+    log_timestamp_and_log("wsrep", severity, msg);
+}
+
+void
+node_log(wsrep_log_level_t const severity,
+         const char*       const file,
+         const char*       const function,
+         int               const line,
+         ...)
+{
+    va_list ap;
+
+    char   string[2048];
+    int    max_string = sizeof(string);
+    char*  str = string;
+
+    /* provide file:func():line info only if debug logging is on */
+    if (NODE_DO_LOG_DEBUG) {
+        int const len = snprintf(str, (size_t)max_string, "%s:%s():%d: ",
+                                 file, function, line);
+        str += len;
+        max_string -= len;
+    }
+
+    va_start(ap, line);
+    {
+        const char* format = va_arg (ap, const char*);
+
+        if (max_string > 0 && NULL != format) {
+            vsnprintf (str, (size_t)max_string, format, ap);
+        }
+    }
+    va_end(ap);
+
+    /* actual logging */
+    log_timestamp_and_log(" node", severity, string);
+}

examples/node/log.h

@@ -0,0 +1,69 @@
+/* Copyright (c) 2019, Codership Oy. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ */
+
+/**
+ * @file This unit defines logging macros for the application and
+ *       a logger callback for the wsrep provider.
+ */
+
+#ifndef NODE_LOG_H
+#define NODE_LOG_H
+
+#include "../../wsrep_api.h"
+
+/**
+ * REPLICATION: a logger callback for wsrep provider
+ */
+extern void
+node_log_cb(wsrep_log_level_t severity, const char* message);
+
+/**
+ * Applicaton log function intended to be used through the macros defined below.
+ * For simplicity it uses log levels defined by wsrep API, but it does not have
+ * to. */
+extern void
+node_log (wsrep_log_level_t level,
+          const char*       file,
+          const char*       function,
+          const int         line,
+          ...);
+
+/**
+ * This variable made global to avoid calling node_log() when debug logging
+ * is disabled. */
+extern wsrep_log_level_t node_log_max_level;
+#define NODE_DO_LOG_DEBUG (WSREP_LOG_DEBUG <= node_log_max_level)
+
+/**
+ * Base logging macro that records current file, function and line number */
+#define NODE_LOG(level, ...)\
+        node_log(level, __FILE__, __func__, __LINE__, __VA_ARGS__, NULL)
+
+/**
+ * @name Logging macros.
+ * Must be implemented as macros to report the location of the code where
+ * they are called.
+ */
+/*@{*/
+#define NODE_FATAL(...) NODE_LOG(WSREP_LOG_FATAL, __VA_ARGS__, NULL)
+#define NODE_ERROR(...) NODE_LOG(WSREP_LOG_ERROR, __VA_ARGS__, NULL)
+#define NODE_WARN(...)  NODE_LOG(WSREP_LOG_WARN,  __VA_ARGS__, NULL)
+#define NODE_INFO(...)  NODE_LOG(WSREP_LOG_INFO,  __VA_ARGS__, NULL)
+#define NODE_DEBUG(...) if (NODE_DO_LOG_DEBUG) \
+    { NODE_LOG(WSREP_LOG_DEBUG, __VA_ARGS__, NULL); }
+/*@}*/
+
+#endif /* NODE_LOG_H */