Be able to bind statefulsets to the host network

hack/docker/redis-tools/redis-tools.sh

@@ -12,6 +12,7 @@ show_help() {
   echo "-h, --help                         show brief help"
   echo "    --data-dir=DIR                 path to directory holding db data (default: /var/data)"
   echo "    --host=HOST                    database host"
+  echo "    --port=PORT                    database port"
   echo "    --user=USERNAME                database username"
   echo "    --bucket=BUCKET                name of bucket"
   echo "    --location=LOCATION            location of backend (<provider>:<bucket name>)"
@@ -50,6 +51,10 @@ while test $# -gt 0; do
       export REDIS_HOST=$(echo $1 | sed -e 's/^[^=]*=//g')
       shift
       ;;
+    --port*)
+      export REDIS_PORT=$(echo $1 | sed -e 's/^[^=]*=//g')
+      shift
+      ;;
     --user*)
       export REDIS_USER=$(echo $1 | sed -e 's/^[^=]*=//g')
       shift
@@ -108,8 +113,8 @@ case "$op" in
     # cleanup data dump dir
     rm -rf *
 
-    redis-cli --rdb dump.rdb -h "${REDIS_HOST}" -a "${REDIS_PASSWORD}"
-    redis-cli -h "${REDIS_HOST}" -a "${REDIS_PASSWORD}" CLUSTER NODES | grep myself > nodes.conf
+    redis-cli --rdb dump.rdb -h "${REDIS_HOST}" -p "${REDIS_PORT}" -a "${REDIS_PASSWORD}"
+    redis-cli -h "${REDIS_HOST}" -p "${REDIS_PORT}" -a "${REDIS_PASSWORD}" CLUSTER NODES | grep myself > nodes.conf
     pwd
     ls -lh "$SOURCE_DIR"
     echo "Uploading dump file to the backend......."

make.Dockerfile

@@ -0,0 +1,27 @@
+FROM golang:1.13.3 as go-builder
+
+RUN apt-get update && apt-get -y upgrade && \
+    apt-get install -y ca-certificates git mercurial
+
+ARG PROJECT_NAME=redis-cluster-operator
+ARG REPO_PATH=github.com/ucloud/$PROJECT_NAME
+ARG BUILD_PATH=${REPO_PATH}/cmd/manager
+
+# Build version and commit should be passed in when performing docker build
+ARG VERSION=0.1.1
+ARG GIT_SHA=0000000
+
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY pkg ./ cmd ./ version ./
+
+#RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o ${GOBIN}/${PROJECT_NAME} \
+#    -ldflags "-X ${REPO_PATH}/version.Version=${VERSION} -X ${REPO_PATH}/version.GitSHA=${GIT_SHA}" \
+#    $BUILD_PATH
+
+COPY . ./
+RUN apt-get install -y bash git make cmake gcc openssh-client openssh-server
+RUN apt-get install -y libc-dev
+RUN make build

pkg/apis/redis/v1alpha1/default.go

@@ -16,6 +16,7 @@ const (
 	minClusterReplicas  = 1
 	defaultRedisImage   = "redis:5.0.4-alpine"
 	defaultMonitorImage = "oliver006/redis_exporter:latest"
+	defaultClientPort   = 6379
 )
 
 func (in *DistributedRedisCluster) DefaultSpec(log logr.Logger) bool {
@@ -25,6 +26,16 @@ func (in *DistributedRedisCluster) DefaultSpec(log logr.Logger) bool {
 		update = true
 	}
 
+	if in.Spec.ClientPort == 0 {
+		in.Spec.ClientPort = defaultClientPort
+		update = true
+	}
+
+	if in.Spec.GossipPort != (in.Spec.ClientPort + 10000) {
+		in.Spec.GossipPort = in.Spec.ClientPort + 10000
+		update = true
+	}
+
 	if in.Spec.Image == "" {
 		in.Spec.Image = defaultRedisImage
 		update = true

pkg/apis/redis/v1alpha1/distributedrediscluster_types.go

@@ -25,18 +25,21 @@ type DistributedRedisClusterSpec struct {
 	ServiceName      string                        `json:"serviceName,omitempty"`
 	Config           map[string]string             `json:"config,omitempty"`
 	// Set RequiredAntiAffinity to force the master-slave node anti-affinity.
-	RequiredAntiAffinity 		bool                         `json:"requiredAntiAffinity,omitempty"`
-	Affinity             		*corev1.Affinity             `json:"affinity,omitempty"`
-	NodeSelector         		map[string]string            `json:"nodeSelector,omitempty"`
-	ToleRations          		[]corev1.Toleration          `json:"toleRations,omitempty"`
-	SecurityContext      		*corev1.PodSecurityContext   `json:"securityContext,omitempty"`
-	ContainerSecurityContext	*corev1.SecurityContext		 `json:"containerSecurityContext,omitempty"`
-	Annotations          		map[string]string            `json:"annotations,omitempty"`
-	Storage              		*RedisStorage                `json:"storage,omitempty"`
-	Resources            		*corev1.ResourceRequirements `json:"resources,omitempty"`
-	PasswordSecret       		*corev1.LocalObjectReference `json:"passwordSecret,omitempty"`
-	Monitor              		*AgentSpec                   `json:"monitor,omitempty"`
-	Init                 		*InitSpec                    `json:"init,omitempty"`
+	RequiredAntiAffinity     	bool                         `json:"requiredAntiAffinity,omitempty"`
+	Affinity                 	*corev1.Affinity             `json:"affinity,omitempty"`
+	NodeSelector             	map[string]string            `json:"nodeSelector,omitempty"`
+	ToleRations              	[]corev1.Toleration          `json:"toleRations,omitempty"`
+	SecurityContext          	*corev1.PodSecurityContext   `json:"securityContext,omitempty"`
+	ContainerSecurityContext 	*corev1.SecurityContext      `json:"containerSecurityContext,omitempty"`
+	Annotations              	map[string]string            `json:"annotations,omitempty"`
+	Storage                  	*RedisStorage                `json:"storage,omitempty"`
+	Resources                	*corev1.ResourceRequirements `json:"resources,omitempty"`
+	PasswordSecret           	*corev1.LocalObjectReference `json:"passwordSecret,omitempty"`
+	Monitor                  	*AgentSpec                   `json:"monitor,omitempty"`
+	Init                     	*InitSpec                    `json:"init,omitempty"`
+	HostNetwork              	bool                         `json:"hostNetwork,omitempty"`
+	ClientPort               	int                          `json:"clientPort,omitempty"`
+	GossipPort               	int                          `json:"gossipPort,omitempty"`
 }
 
 type AgentSpec struct {

pkg/controller/distributedrediscluster/distributedrediscluster_controller.go

@@ -232,13 +232,13 @@ func (r *ReconcileDistributedRedisCluster) Reconcile(request reconcile.Request)
 		return reconcile.Result{}, Kubernetes.Wrap(err, "getClusterPassword")
 	}
 
-	admin, err := newRedisAdmin(ctx.pods, password, config.RedisConf(), reqLogger)
+	admin, err := newRedisAdmin(ctx.pods, password, config.RedisConf(), reqLogger, instance.Spec.ClientPort)
 	if err != nil {
 		return reconcile.Result{}, Redis.Wrap(err, "newRedisAdmin")
 	}
 	defer admin.Close()
 
-	clusterInfos, err := admin.GetClusterInfos()
+	clusterInfos, err := admin.GetClusterInfos(ctx.cluster.Spec.ClientPort)
 	if err != nil {
 		if clusterInfos.Status == redisutil.ClusterInfosPartial {
 			return reconcile.Result{}, Redis.Wrap(err, "GetClusterInfos")
@@ -331,8 +331,7 @@ func (r *ReconcileDistributedRedisCluster) Reconcile(request reconcile.Request)
 			return reconcile.Result{}, err
 		}
 	}
-
-	newClusterInfos, err := admin.GetClusterInfos()
+	newClusterInfos, err := admin.GetClusterInfos(ctx.cluster.Spec.ClientPort)
 	if err != nil {
 		if clusterInfos.Status == redisutil.ClusterInfosPartial {
 			return reconcile.Result{}, Redis.Wrap(err, "GetClusterInfos")

pkg/controller/distributedrediscluster/helper.go

@@ -3,6 +3,7 @@ package distributedrediscluster
 import (
 	"fmt"
 	"net"
+	"strconv"
 	"time"
 
 	"github.com/go-logr/logr"
@@ -29,10 +30,10 @@ func getLabels(cluster *redisv1alpha1.DistributedRedisCluster) map[string]string
 }
 
 // newRedisAdmin builds and returns new redis.Admin from the list of pods
-func newRedisAdmin(pods []*corev1.Pod, password string, cfg *config.Redis, reqLogger logr.Logger) (redisutil.IAdmin, error) {
+func newRedisAdmin(pods []*corev1.Pod, password string, cfg *config.Redis, reqLogger logr.Logger, clientPort int) (redisutil.IAdmin, error) {
 	nodesAddrs := []string{}
 	for _, pod := range pods {
-		redisPort := redisutil.DefaultRedisPort
+		redisPort := strconv.Itoa(clientPort)
 		for _, container := range pod.Spec.Containers {
 			if container.Name == "redis" {
 				for _, port := range container.Ports {

pkg/controller/distributedrediscluster/sync_handler.go

@@ -161,7 +161,7 @@ func (r *ReconcileDistributedRedisCluster) initRestore(cluster *redisv1alpha1.Di
 }
 
 func (r *ReconcileDistributedRedisCluster) waitForClusterJoin(ctx *syncContext) error {
-	if infos, err := ctx.admin.GetClusterInfos(); err == nil {
+	if infos, err := ctx.admin.GetClusterInfos(ctx.cluster.Spec.ClientPort); err == nil {
 		ctx.reqLogger.V(6).Info("debug waitForClusterJoin", "cluster infos", infos)
 		return nil
 	}
@@ -180,7 +180,7 @@ func (r *ReconcileDistributedRedisCluster) waitForClusterJoin(ctx *syncContext)
 	// the config as they are still empty with unassigned slots.
 	time.Sleep(1 * time.Second)
 
-	_, err = ctx.admin.GetClusterInfos()
+	_, err = ctx.admin.GetClusterInfos(ctx.cluster.Spec.ClientPort)
 	if err != nil {
 		return Requeue.Wrap(err, "wait for cluster join")
 	}
@@ -278,7 +278,7 @@ func (r *ReconcileDistributedRedisCluster) scalingDown(ctx *syncContext, current
 			if len(node.Slots) > 0 {
 				return Redis.New(fmt.Sprintf("node %s is not empty! Reshard data away and try again", node.String()))
 			}
-			if err := admin.ForgetNode(node.ID); err != nil {
+			if err := admin.ForgetNode(node.ID,ctx.cluster.Spec.ClientPort); err != nil {
 				return Redis.Wrap(err, "ForgetNode")
 			}
 		}
@@ -346,7 +346,7 @@ func (r *ReconcileDistributedRedisCluster) resetClusterPassword(ctx *syncContext
 		}
 
 		podSet := clusterPods(redisClusterPods.Items)
-		admin, err := newRedisAdmin(podSet, oldPassword, config.RedisConf(), ctx.reqLogger)
+		admin, err := newRedisAdmin(podSet, oldPassword, config.RedisConf(), ctx.reqLogger, ctx.cluster.Spec.ClientPort)
 		if err != nil {
 			return err
 		}

pkg/controller/heal/failednodes.go

@@ -17,7 +17,7 @@ func (c *CheckAndHeal) FixFailedNodes(cluster *redisv1alpha1.DistributedRedisClu
 		c.Logger.Info("[FixFailedNodes] Forgetting failed node, this command might fail, this is not an error", "node", id)
 		if !c.DryRun {
 			c.Logger.Info("[FixFailedNodes] try to forget node", "nodeId", id)
-			if err := admin.ForgetNode(id); err != nil {
+			if err := admin.ForgetNode(id, cluster.Spec.ClientPort); err != nil {
 				errs = append(errs, err)
 			}
 		}

pkg/controller/heal/untrustenodes.go

@@ -43,7 +43,7 @@ func (c *CheckAndHeal) FixUntrustedNodes(cluster *redisv1alpha1.DistributedRedis
 		doneAnAction = true
 		if !c.DryRun {
 			c.Logger.Info("[FixUntrustedNodes] try to forget node", "nodeId", id)
-			if err := admin.ForgetNode(id); err != nil {
+			if err := admin.ForgetNode(id,cluster.Spec.ClientPort); err != nil {
 				errs = append(errs, err)
 			}
 		}

pkg/controller/redisclusterbackup/sync_handler.go

@@ -326,6 +326,7 @@ func (r *ReconcileRedisClusterBackup) backupContainers(backup *redisv1alpha1.Red
 					fmt.Sprintf(`--data-dir=%s`, redisv1alpha1.BackupDumpDir),
 					fmt.Sprintf(`--location=%s`, location),
 					fmt.Sprintf(`--host=%s`, node.IP),
+					fmt.Sprintf(`--port=%s`, node.Port),
 					fmt.Sprintf(`--folder=%s`, folderName),
 					fmt.Sprintf(`--snapshot=%s-%d`, backup.Name, i),
 					"--",

pkg/osm/osm.go

@@ -80,69 +80,75 @@ func NewOSMContext(client client.Client, spec api.Backend, namespace string) (*o
 
 	if spec.S3 != nil {
 		nc.Provider = s3.Kind
-
 		keyID, foundKeyID := config[awsconst.AWS_ACCESS_KEY_ID]
 		key, foundKey := config[awsconst.AWS_SECRET_ACCESS_KEY]
-		if foundKey && foundKeyID {
+		if spec.S3.Endpoint == "" || spec.S3.Endpoint == "osm" {
 			nc.Config[s3.ConfigAccessKeyID] = string(keyID)
 			nc.Config[s3.ConfigSecretKey] = string(key)
 			nc.Config[s3.ConfigAuthType] = "accesskey"
+			nc.Config[s3.ConfigRegion] = spec.S3.Region
 		} else {
-			nc.Config[s3.ConfigAuthType] = "iam"
-		}
-		if spec.S3.Endpoint == "" || strings.HasSuffix(spec.S3.Endpoint, ".amazonaws.com") {
-			// Using s3 and not s3-compatible service like minio or rook, etc. Now, find region
-			var sess *session.Session
-			var err error
-			if nc.Config[s3.ConfigAuthType] == "iam" {
-				// The aws sdk does not currently support automatically setting the region based on an instances placement.
-				// This automatically sets region based on ec2 instance metadata when running on EC2.
-				// ref: https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/setting-region.html#setting-region-order-of-precedence
-				var c aws.Config
-				if s, e := session.NewSession(); e == nil {
-					if region, e := ec2metadata.New(s).Region(); e == nil {
-						c.WithRegion(region)
+			if foundKey && foundKeyID {
+				nc.Config[s3.ConfigAccessKeyID] = string(keyID)
+				nc.Config[s3.ConfigSecretKey] = string(key)
+				nc.Config[s3.ConfigAuthType] = "accesskey"
+			} else {
+				nc.Config[s3.ConfigAuthType] = "iam"
+			}
+			if spec.S3.Endpoint == "" || strings.HasSuffix(spec.S3.Endpoint, ".amazonaws.com") {
+				// Using s3 and not s3-compatible service like minio or rook, etc. Now, find region
+				var sess *session.Session
+				var err error
+				if nc.Config[s3.ConfigAuthType] == "iam" {
+					// The aws sdk does not currently support automatically setting the region based on an instances placement.
+					// This automatically sets region based on ec2 instance metadata when running on EC2.
+					// ref: https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/setting-region.html#setting-region-order-of-precedence
+					var c aws.Config
+					if s, e := session.NewSession(); e == nil {
+						if region, e := ec2metadata.New(s).Region(); e == nil {
+							c.WithRegion(region)
+						}
 					}
+					sess, err = session.NewSessionWithOptions(session.Options{
+						Config: c,
+						// Support MFA when authing using assumed roles.
+						SharedConfigState:       session.SharedConfigEnable,
+						AssumeRoleTokenProvider: stscreds.StdinTokenProvider,
+					})
+				} else {
+					sess, err = session.NewSessionWithOptions(session.Options{
+						Config: aws.Config{
+							Credentials: credentials.NewStaticCredentials(string(keyID), string(key), ""),
+							Region:      aws.String("us-east-1"),
+						},
+						// Support MFA when authing using assumed roles.
+						SharedConfigState:       session.SharedConfigEnable,
+						AssumeRoleTokenProvider: stscreds.StdinTokenProvider,
+					})
 				}
-				sess, err = session.NewSessionWithOptions(session.Options{
-					Config: c,
-					// Support MFA when authing using assumed roles.
-					SharedConfigState:       session.SharedConfigEnable,
-					AssumeRoleTokenProvider: stscreds.StdinTokenProvider,
+				if err != nil {
+					return nil, err
+				}
+				svc := _s3.New(sess)
+				out, err := svc.GetBucketLocation(&_s3.GetBucketLocationInput{
+					Bucket: types.StringP(spec.S3.Bucket),
 				})
+				if err != nil {
+					return nil, err
+				}
+				nc.Config[s3.ConfigRegion] = stringz.Val(types.String(out.LocationConstraint), "us-east-1")
 			} else {
-				sess, err = session.NewSessionWithOptions(session.Options{
-					Config: aws.Config{
-						Credentials: credentials.NewStaticCredentials(string(keyID), string(key), ""),
-						Region:      aws.String("us-east-1"),
-					},
-					// Support MFA when authing using assumed roles.
-					SharedConfigState:       session.SharedConfigEnable,
-					AssumeRoleTokenProvider: stscreds.StdinTokenProvider,
-				})
-			}
-			if err != nil {
-				return nil, err
-			}
-			svc := _s3.New(sess)
-			out, err := svc.GetBucketLocation(&_s3.GetBucketLocationInput{
-				Bucket: types.StringP(spec.S3.Bucket),
-			})
-			if err != nil {
-				return nil, err
-			}
-			nc.Config[s3.ConfigRegion] = stringz.Val(types.String(out.LocationConstraint), "us-east-1")
-		} else {
-			nc.Config[s3.ConfigEndpoint] = spec.S3.Endpoint
-			u, err := url.Parse(spec.S3.Endpoint)
-			if err != nil {
-				return nil, err
-			}
-			nc.Config[s3.ConfigDisableSSL] = strconv.FormatBool(u.Scheme == "http")
+				nc.Config[s3.ConfigEndpoint] = spec.S3.Endpoint
+				u, err := url.Parse(spec.S3.Endpoint)
+				if err != nil {
+					return nil, err
+				}
+				nc.Config[s3.ConfigDisableSSL] = strconv.FormatBool(u.Scheme == "http")
 
-			cacertData, ok := config[awsconst.CA_CERT_DATA]
-			if ok && u.Scheme == "https" {
-				nc.Config[s3.ConfigCACertData] = string(cacertData)
+				cacertData, ok := config[awsconst.CA_CERT_DATA]
+				if ok && u.Scheme == "https" {
+					nc.Config[s3.ConfigCACertData] = string(cacertData)
+				}
 			}
 		}
 		return nc, nil