Enable leader election for the operator by default. (#367)

Author: HoustonPutman

docs/running-the-operator.md

@@ -126,10 +126,28 @@ The final image will only contain the solr-operator binary and necessary License
 
 ## Solr Operator Input Args
 
-* **-zookeeper-operator** Whether or not to use the Zookeeper Operator to create dependency Zookeeepers.
-                          Required to use the `spec.zookeeperRef.provided` option.
-                          If _true_, then a Zookeeper Operator must be running for the cluster.
-                          (_true_ | _false_ , defaults to _false_)
+* **--zookeeper-operator** Whether or not to use the Zookeeper Operator to create dependency Zookeeepers.
+  Required to use the `spec.zookeeperRef.provided` option.
+  If _true_, then a Zookeeper Operator must be running for the cluster.
+  (_true_ | _false_ , defaults to _false_)
+
+* **--watch-namespaces** Watch certain namespaces in the Kubernetes cluster.
+  If flag is omitted, or given an empty string, then the whole cluster will be watched.
+  If the operator should watch multiple namespaces, provide them all separated by commas.
+  (_string_ , defaults to _empty_)
+
+* **--leader-elect** Whether or not to use leader election for the Solr Operator.
+  If set to true, then only one operator pod will be functional for the namespaces given through `--watch-namespaces`.
+  If multiple namespaces are provided, leader election will use the first namespace sorted alphabetically.
+  (_true_ | _false_ , defaults to _true_)
+
+* **--metrics-bind-address** The address to bind the metrics servlet on.
+  If only a port is provided (e.g. `:8080`), then the metrics server will respond to requests with any Host header.
+  (defaults to _:8080_)
+
+* **--health-probe-bind-address=** The address to bind the health probe servlet on.
+  If only a port is provided (e.g. `:8081`), then the metrics server will respond to requests with any Host header.
+  (defaults to _:8081_)
 
 ## Client Auth for mTLS-enabled Solr clusters
 

helm/solr-operator/Chart.yaml

@@ -192,6 +192,13 @@ annotations:
           url: https://github.com/apache/solr-operator/pull/359
         - name: SolrBackup Documentation
           url: https://apache.github.io/solr-operator/docs/solr-backup#recurring-backups
+    - kind: changed
+      description: Solr Operator Leader Election enabled by default
+      links:
+        - name: Github Issue
+          url: https://github.com/apache/solr-operator/issues/366
+        - name: Github PR
+          url: https://github.com/apache/solr-operator/pull/367
   artifacthub.io/images: |
     - name: solr-operator
       image: apache/solr-operator:v0.5.0-prerelease

helm/solr-operator/README.md

@@ -158,8 +158,9 @@ The command removes all the Kubernetes components associated with the chart and
 |-----|------|---------|-------------|
 | watchNamespaces | string | `""` | A comma-separated list of namespaces that the solr operator should watch. If empty, the solr operator will watch all namespaces in the cluster. If set to `true`, this will be populated with the namespace that the operator is deployed to. |
 | zookeeper-operator.install | boolean | `true` | This option installs the Zookeeper Operator as a helm dependency |
-| metrics.enable | boolean | `true` | Enable metrics for the Solr Operator. Will be available via the "metrics"/8080 port on the solr operator pods under the "/metrics" path. |
 | zookeeper-operator.use | boolean | `false` | This option enables the use of provided Zookeeper instances for SolrClouds via the Zookeeper Operator, without installing the Zookeeper Operator as a dependency. If `zookeeper-operator.install`=`true`, then this option is ignored. |
+| leaderElection.enable | boolean | `true` | Enable leader election for the Solr Operator. Will work across multiple `watchNamespaces`, as long as all deployments have the same list for `watchNamespaces`. |
+| metrics.enable | boolean | `true` | Enable metrics for the Solr Operator. Will be available via the "metrics"/8080 port on the solr operator pods under the "/metrics" path. |
 | mTLS.clientCertSecret | string | `""` | Name of a Kubernetes TLS secret, in the same namespace, that contains a Client certificate to load into the operator. If provided, this is used when communicating with Solr. |
 | mTLS.caCertSecretKey | string | `""` | Name of a Kubernetes secret, in the same namespace, that contains PEM encoded Root CA Certificate to use when connecting to Solr with Client Auth. |
 | mTLS.caCertSecret | string | `""` | Name of the key in the `caCertSecret` that contains the Root CA Cert as a value. |

helm/solr-operator/templates/deployment.yaml

@@ -71,6 +71,9 @@ spec:
         {{- if .Values.metrics.enable }}
         - "--metrics-bind-address=:8080"
         {{- end }}
+        {{- if not .Values.leaderElection.enable }}
+        - "--leader-elect=false"
+        {{- end }}
 
         env:
           - name: POD_NAMESPACE

helm/solr-operator/values.yaml

@@ -27,6 +27,9 @@ image:
 nameOverride: ""
 fullnameOverride: ""
 
+leaderElection:
+  enable: true
+
 zookeeper-operator:
   # Include the zookeeper-operator as a helm chart dependency.
   # Setting this to true also tells the Solr Operator it is use to use ZookeeperCluster objects.

main.go

@@ -32,6 +32,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sort"
 	"strings"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
@@ -114,7 +115,7 @@ func main() {
 	var probeAddr string
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
-	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
+	flag.BoolVar(&enableLeaderElection, "leader-elect", true,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
 	opts := zap.Options{
@@ -135,33 +136,38 @@ func main() {
 	setupLog.Info(fmt.Sprintf("Go Version: %v", runtime.Version()))
 	setupLog.Info(fmt.Sprintf("Go OS/Arch: %s / %s", runtime.GOOS, runtime.GOARCH))
 
-	// When the operator is started to watch resources in a specific set of namespaces, we use the MultiNamespacedCacheBuilder cache.
-	// In this scenario, it is also suggested to restrict the provided authorization to this namespace by replacing the default
-	// ClusterRole and ClusterRoleBinding to Role and RoleBinding respectively
-	// For further information see the kubernetes documentation about
-	// Using [RBAC Authorization](https://kubernetes.io/docs/reference/access-authn-authz/rbac/).
-	var managerWatchCache cache.NewCacheFunc
+	operatorOptions := ctrl.Options{
+		Scheme:                 scheme,
+		MetricsBindAddress:     metricsAddr,
+		Port:                   9443,
+		HealthProbeBindAddress: probeAddr,
+		LeaderElection:         enableLeaderElection,
+		LeaderElectionID:       "88488bdc.solr.apache.org",
+	}
+
+	/*
+		When the operator is started to watch resources in a specific set of namespaces, we use the MultiNamespacedCacheBuilder cache.
+		In this scenario, it is also suggested to restrict the provided authorization to this namespace by replacing the default
+		ClusterRole and ClusterRoleBinding to Role and RoleBinding respectively
+		For further information see the kubernetes documentation about
+		Using [RBAC Authorization](https://kubernetes.io/docs/reference/access-authn-authz/rbac/).
+
+		When watching multiple namespaces with leader election enabled, the leader election will use the lowest-sorted namespace.
+		It is likely safer to run individual solr operators per-namespace, to ensure leader election is working as expected no matter what.
+	*/
 	if watchNamespaces != "" {
 		setupLog.Info(fmt.Sprintf("Managing for Namespaces: %s", watchNamespaces))
 		ns := strings.Split(watchNamespaces, ",")
 		for i := range ns {
 			ns[i] = strings.TrimSpace(ns[i])
 		}
-		managerWatchCache = cache.MultiNamespacedCacheBuilder(ns)
-	} else {
-		setupLog.Info("Managing for the entire cluster.")
-		managerWatchCache = (cache.NewCacheFunc)(nil)
+		sort.Strings(ns)
+		operatorOptions.NewCache = cache.MultiNamespacedCacheBuilder(ns)
+
+		operatorOptions.LeaderElectionNamespace = ns[0]
 	}
 
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme:                 scheme,
-		MetricsBindAddress:     metricsAddr,
-		Port:                   9443,
-		HealthProbeBindAddress: probeAddr,
-		LeaderElection:         enableLeaderElection,
-		LeaderElectionID:       "88488bdc.solr.apache.org",
-		NewCache:               managerWatchCache,
-	})
+	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), operatorOptions)
 	if err != nil {
 		setupLog.Error(err, "unable to start solr operator")
 		os.Exit(1)