Release version 0.9.

- Implement deterministic failover
- Based on the version in ganesha-ha

Author: jarrpa

src/storhaug

@@ -37,6 +37,7 @@ HA_SMB_MNT_DIR="lock"
 HA_NFS_MNT_DIR="state"
 STORAGE_SERVERS=""
 STORAGE_NUM_SERVERS=0
+DETERMINISTIC_FAILOVER=false
 
 ### Utility functions
 
@@ -61,6 +62,15 @@ usage()
     echo -e "  ${HA_CONF_INCDIR}*.conf"
 }
 
+parsebool()
+{
+    case $(eval echo \${${1}}) in
+        TRUE | True | true | YES | Yes | yes) declare "${1}"=true ;;
+        FALSE | False | false | NO | No | no) declare "${1}"=false ;;
+        *) storlog "ERR" "Couldn't parse boolean: ${1}=$(eval echo \${${1}})" ;;
+    esac
+}
+
 storlog()
 {
     LEVEL=$1; shift
@@ -116,9 +126,14 @@ determine_servers()
         IFS=${tmp_ifs}
         HA_NUM_SERVERS=${num_servers}
         HA_SERVERS="${ha_servers}"
-        # TODO: Determine storage servers from pcs status
-        STORAGE_SERVERS="${STORAGE_NODES//,/ }"
-        STORAGE_NUM_SERVERS=$(wc -w <<< "${STORAGE_SERVERS}")
+        # TODO: Determine storage and vip servers from pcs status
+        if [[ "x${STORAGE_NODES}" != "x" ]]; then
+            STORAGE_SERVERS="${STORAGE_NODES//,/ }"
+            STORAGE_NUM_SERVERS=$(wc -w <<< "${STORAGE_SERVERS}")
+        fi
+        if [[ "x${VIP_NODES}" != "x" ]]; then
+            VIP_SERVERS="${VIP_NODES//,/ }"
+        fi
     else
         IFS=$','
         for server in ${HA_CLUSTER_NODES} ; do
@@ -127,8 +142,13 @@ determine_servers()
         IFS=${tmp_ifs}
         HA_NUM_SERVERS=${num_servers}
         HA_SERVERS="${HA_CLUSTER_NODES//,/ }"
-        STORAGE_SERVERS="${STORAGE_NODES//,/ }"
-        STORAGE_NUM_SERVERS=$(wc -w <<< "${STORAGE_SERVERS}")
+        if [[ "x${STORAGE_NODES}" != "x" ]]; then
+            STORAGE_SERVERS="${STORAGE_NODES//,/ }"
+            STORAGE_NUM_SERVERS=$(wc -w <<< "${STORAGE_SERVERS}")
+        fi
+        if [[ "x${VIP_NODES}" != "x" ]]; then
+            VIP_SERVERS="${VIP_NODES//,/ }"
+        fi
     fi
 }
 
@@ -194,6 +214,65 @@ teardown_cluster()
     pcs cluster destroy || storlog "ERR" "Failed to destroy cluster ${name}"
 }
 
+### Resource functions
+
+do_create_virt_ip_constraints()
+{
+    local cibfile=${1}; shift
+    local ipcount=${1}; shift
+    local primary=${1}; shift
+    local weight="1000"
+
+    # A set of location constraints to set the prefered order
+    # for where a VIP should move
+    while [[ ${1} ]]; do
+        pcs -f ${cibfile} constraint location vip${ipcount} prefers ${1}=${weight} || \
+          storlog "WARN" "Failed: pcs constraint location vip${ipcount} prefers ${1}=${weight}"
+        weight=$(expr ${weight} + 1000)
+        shift
+    done
+
+    # Set the highest preference for the VIP to its primary node
+    pcs -f ${cibfile} constraint location vip${ipcount} prefers ${primary}=${weight} || \
+      storlog "WARN" "Failed: pcs constraint location vip${ipcount} prefers ${primary}=${weight}"
+}
+
+wrap_create_virt_ip_constraints()
+{
+    local cibfile=${1}; shift
+    local ipcount=${1}; shift
+    local srvcount=${ipcount}
+    local primary=""
+    local head=""
+    local tail=""
+
+    # build a list of failover peers, e.g. for a four node cluster, for node1,
+    # the result is "node2 node3 node4"; for node2, "node3 node4 node1"
+    # and so on.
+    read -r -a servers <<< "${VIP_SERVERS:-STORAGE_SERVERS}"
+    while [ ${srvcount} -gt ${STORAGE_NUM_SERVERS} ]; do
+        srvcount=$((srvcount - STORAGE_NUM_SERVERS))
+    done
+    primary=${servers[${srvcount}-1]}
+    if [ ${STORAGE_NUM_SERVERS} -gt 1 ]; then
+        head=${servers[@]:${srvcount}-${STORAGE_NUM_SERVERS}-1}
+        tail=${servers[@]:${srvcount}}
+    fi
+
+    do_create_virt_ip_constraints ${cibfile} ${ipcount} ${primary} ${tail} ${head}
+}
+
+create_virt_ip_constraints()
+{
+    local cibfile=${1}; shift
+    local ipcount=0
+
+    for ip in ${HA_VIPS}; do
+        ((ipcount++))
+        wrap_create_virt_ip_constraints ${cibfile} ${ipcount}
+        shift
+    done
+}
 
 setup_create_resources()
 {
@@ -293,6 +372,10 @@ setup_create_resources()
 #        pcs -f ${cibfile} constraint order vip${ipcount} then vip${ipcount}_trigger
     done
 
+    if [[ ${DETERMINISTIC_FAILOVER} == true ]]; then
+        create_virt_ip_constraints ${cibfile}
+    fi
+
     pcs cluster cib-push ${cibfile} || storlog "ERR" "Failed to create virtual IP resources."
 
     rm -f ${cibfile}
@@ -357,6 +440,8 @@ done
 # Source/load the config
 . $HA_CONF_sec
 
+parsebool "DETERMINISTIC_FAILOVER"
+
 case "${cmd}" in
     setup | --setup)
         storlog "INFO" "Setting up ${HA_NAME}"

src/storhaug.conf.sample

@@ -10,11 +10,11 @@ HA_SERVER="server1"
 
 # The set of nodes that forms the HA cluster.
 # Comma-deliminated IP/Hostname list
-HA_CLUSTER_NODES="server1,server2,..."
+HA_CLUSTER_NODES="server1,server2,server3,..."
 
 # [OPTIONAL] A subset of HA nodes that will serve as storage servers.
 # Comma-deliminated IP/Hostname list
-STORAGE_NODES="server2,..."
+STORAGE_NODES="server2,server3,..."
 
 # [OPTIONAL] Mount point for shared volumes used by HA resources.
 HA_MNT_DIR="/var/run/gluster"
@@ -23,6 +23,14 @@ HA_MNT_DIR="/var/run/gluster"
 # Whitespace-deliminated IP address list
 HA_VIPS="10.x.x.x 10.x.x.x"
 
+# Deterministic Failover: If true, configures virtual IP failover to occur in
+# a deterministic order. The optional VIP_NODES setting specifies which node
+# will serve as the primary/home node for the corresponding virtual IP address.
+#   NOTE: These nodes should also be configured as storage nodes, either
+#         implicitly in HA_CLUSTER_NODES or explicitly in STORAGE_NODES
+DETERMINISTIC_FAILOVER=false
+VIP_NODES="server2,server3"
+
 # Managed access methods
 # Whitespace-delimited list. Valid values:
 #   nfs