Fix the development environment for linux, darwin and M1 arch

Author: prafull01

DEVELOPER.md

@@ -44,6 +44,13 @@ since there's no need to set up a remote GKE/OpenShift cluster.
 
 **make dev/up**
 
+Pre-requisite: The command sets up a k3d registry, and to push an image to it, the registry name must resolve to 
+localhost or 127.0.0.1. Please add the following line to your /etc/hosts file:
+
+```shell
+127.0.0.1 registry.localhost
+```
+
 This command will get everything set up for you to begin testing out the operator. Specifically it will:
 
 * Start a k3d cluster named test (context=k3d-test) with a managed docker registry
@@ -56,6 +63,18 @@ This command will get everything set up for you to begin testing out the operato
 
 Tears down the k3d cluster.
 
+[Existing Cluster & Repo]:
+
+export DEV_REGISTRY="Your_Registry"
+**make k8s/apply**
+
+This command will deploy the operator to an existing cluster. It will build the operator image and push it to the DEV_REGISTRY.
+This newly built image will be used to deploy the operator.
+
+**make k8s/delete**
+
+Tears down the cockroach operator deployment applied on the cluster.
+
 ## Testing CR Database
 
 Notes on how to test an existing CR Database.
@@ -71,6 +90,9 @@ SELECT * FROM bank.accounts;
 
 ## Developer Install Instructions
 
+Pre-requisite: 
+1. Install kustomize
+
 These instructions are for developers only. If you want to try the alpha please use the instructions in the next
 section.
 
@@ -79,13 +101,18 @@ Install the operator
 ```console
 $ git clone https://github.com/cockroachdb/cockroach-operator.git
 $ export CLUSTER=test
+$ export APP_VERSION=v$(cat version.txt)
+$ export DEV_REGISTRY=us.gcr.io/$(gcloud config get-value project)
 # create a gke cluster
 $ ./hack/create-gke-cluster.sh -c $CLUSTER
 
-$ DEV_REGISTRY=us.gcr.io/$(gcloud config get-value project) \
+$ bazel run //hack/crdbversions:crdbversions -- -operator-image ${DEV_REGISTRY}/cockroach-operator \
+-operator-version ${APP_VERSION} -crdb-versions $(PWD)/crdb-versions.yaml -repo-root $(PWD)
+ 
+$ DEV_REGISTRY=${DEV_REGISTRY} \
 K8S_CLUSTER=$(kubectl config view --minify -o=jsonpath='{.contexts[0].context.cluster}') \
 bazel run --stamp --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 \
-                 //manifests:install_operator.apply
+                 --define APP_VERSION=${APP_VERSION} //config/default:install.apply
 ```
 
 There are various examples that can be installed. The files are located in the examples directory.

Makefile

@@ -221,7 +221,7 @@ test/preflight-%: release/generate-bundle
 #
 .PHONY: dev/build
 dev/build: dev/syncdeps
-	bazel build //...
+	bazel build //... --define APP_VERSION=$(APP_VERSION)
 
 .PHONY: dev/fmt
 dev/fmt:
@@ -280,17 +280,15 @@ dev/down:
 #
 .PHONY: k8s/apply
 k8s/apply:
-	K8S_CLUSTER=gke_$(GCP_PROJECT)_$(GCP_ZONE)_$(CLUSTER_NAME) \
 	DEV_REGISTRY=$(DEV_REGISTRY) \
 	bazel run --stamp --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 \
 		//config/default:install.apply \
 		--define APP_VERSION=$(APP_VERSION)
 
 .PHONY: k8s/delete
 k8s/delete:
-	K8S_CLUSTER=gke_$(GCP_PROJECT)_$(GCP_ZONE)_$(CLUSTER_NAME) \
 	DEV_REGISTRY=$(DEV_REGISTRY) \
-	bazel run --stamp --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 \
+	bazel run --stamp --incompatible_use_cc_configure_from_rules_cc \
 		//config/default:install.delete \
 		--define APP_VERSION=$(APP_VERSION)
 

WORKSPACE

@@ -69,9 +69,9 @@ go_dependencies()
 ######################################
 http_archive(
     name = "io_bazel_rules_docker",
-    sha256 = "1f4e59843b61981a96835dc4ac377ad4da9f8c334ebe5e0bb3f58f80c09735f4",
-    strip_prefix = "rules_docker-0.19.0",
-    urls = ["https://github.com/bazelbuild/rules_docker/releases/download/v0.19.0/rules_docker-v0.19.0.tar.gz"],
+    sha256 = "92779d3445e7bdc79b961030b996cb0c91820ade7ffa7edca69273f404b085d5",
+    strip_prefix = "rules_docker-0.20.0",
+    urls = ["https://github.com/bazelbuild/rules_docker/releases/download/v0.20.0/rules_docker-v0.20.0.tar.gz"],
 )
 
 load(
@@ -103,6 +103,19 @@ container_pull(
     tag = "latest",
 )
 
+http_archive(
+    name = "rules_pkg",
+    sha256 = "8f9ee2dc10c1ae514ee599a8b42ed99fa262b757058f65ad3c384289ff70c4b8",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz",
+        "https://github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz",
+    ],
+)
+
+load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
+
+rules_pkg_dependencies()
+
 ################################
 # Load rules_k8s and configure #
 ################################

cmd/cockroach-operator/BUILD.bazel

@@ -1,7 +1,7 @@
 load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
 load("@io_bazel_rules_docker//container:container.bzl", "container_image")
 load("@io_bazel_rules_docker//go:image.bzl", "go_image")
-load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar")
+load("@rules_pkg//:pkg.bzl", "pkg_tar")
 load("@io_bazel_rules_docker//docker/util:run.bzl", "container_run_and_commit_layer")
 
 go_library(
@@ -55,6 +55,7 @@ container_run_and_commit_layer(
 
 container_image(
     name = "ubi_base_image",
+    architecture = "amd64",
     # References container_pull from WORKSPACE
     base = "@redhat_ubi_minimal//image",
     labels = {

config/default/BUILD.bazel

@@ -32,7 +32,7 @@ k8s_deploy(
     name = "image",
     images = {
         # when running locally, use the image from the local codebase
-        "cockroachdb/cockroach-operator:$(APP_VERSION)": "//cmd/cockroach-operator:operator_image",
+        "cockroach-operator:$(APP_VERSION)": "//cmd/cockroach-operator:operator_image",
     },
     resolver_args = ["--allow_unused_images"],
     template = ":manifest",

config/templates/deployment_image.yaml.in

@@ -24,7 +24,7 @@ spec:
     spec:
       containers:
         - name: cockroach-operator
-          image: cockroachdb/cockroach-operator:{{.OperatorVersion}}
+          image: {{.OperatorImage}}:{{.OperatorVersion}}
           env:
 {{- range .CrdbVersions}}
             {{- /*

hack/bin/BUILD.bazel

@@ -42,6 +42,8 @@ genrule(
     name = "fetch_preflight",
     srcs = select({
         ":k8": ["@preflight_linux//file"],
+        ":m1": ["@preflight_linux//file"],
+        ":darwin": ["@preflight_linux//file"],
     }),
     outs = ["preflight"],
     cmd = "cp $(SRCS) $@",
@@ -75,8 +77,8 @@ genrule(
 genrule(
     name = "io_kubernetes_kube-apiserver",
     srcs = select({
-        ":m1": ["@kube-apiserver_darwin_amd64//file"],
-        ":darwin": ["@kube-apiserver_darwin_amd64//file"],
+        ":m1": ["@kube-apiserver_linux_amd64//file"],
+        ":darwin": ["@kube-apiserver_linux_amd64//file"],
         ":k8": ["@kube-apiserver_linux_amd64//file"],
     }),
     outs = ["kube-apiserver"],

hack/build/repos.bzl

@@ -5336,8 +5336,8 @@ def _go_dependencies():
         build_file_generation = "on",
         build_file_proto_mode = "disable",
         importpath = "gopkg.in/yaml.v3",
-        sum = "h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=",
-        version = "v3.0.0-20210107192922-496545a6307b",
+        sum = "h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=",
+        version = "v3.0.0",
     )
     go_repository(
         name = "io_etcd_go_bbolt",

hack/common.sh

@@ -28,16 +28,22 @@ command -v kubectl >/dev/null 2>&1 || { \
 command -v kustomize >/dev/null 2>&1 || { \
  echo >&2 "I require kustomize but it's not installed.  Aborting."; exit 1; }
 
-usage() { echo "Usage: $0 [-c <cluster name>]" 1>&2; exit 1; }
+usage() { echo "Usage: $0 [-c <cluster name> -r <REGION> -z <ZONE>]" 1>&2; exit 1; }
 # parse -c flag for the CLUSTER_NAME using getopts
-while getopts ":c:i:" opt; do
+while getopts ":c:i:z:r:" opt; do
   case ${opt} in
     c)
       CLUSTER_NAME=$OPTARG
       ;;
     i)
       IMAGE_NAME=$OPTARG
       ;;
+    z)
+      ZONE=$OPTARG
+      ;;
+    r)
+      REGION=$OPTARG
+      ;;
     \?)
       echo "Invalid flag on command line: $OPTARG" 1>&2
       ;;
@@ -55,20 +61,29 @@ if [ -z "${CLUSTER_NAME}" ]; then
     usage
 fi
 
-# Get the default zone and use it or die
-ZONE=$(gcloud config get-value compute/zone)
+# If user did not pass in -z flag then get the default zone and use it or die
 if [ -z "${ZONE}" ]; then
+  # Get the default zone and use it or die
+  ZONE=$(gcloud config get-value compute/zone)
+  if [ -z "${ZONE}" ]; then
     echo "gcloud cli must be configured with a default zone." 1>&2
     echo "run 'gcloud config set compute/zone ZONE'." 1>&2
     echo "replace 'ZONE' with the zone name like us-west1-a." 1>&2
+    echo "Or provide 'ZONE' as input to script as follows:" 1>&2
+    usage
     exit 1;
+  fi
 fi
 
-#Get the default region and use it or die
-REGION=$(gcloud config get-value compute/region)
+#If user did not pass in -r flag then get the default region and use it or die
 if [ -z "${REGION}" ]; then
+  REGION=$(gcloud config get-value compute/region)
+  if [ -z "${REGION}" ]; then
     echo "gcloud cli must be configured with a default region." 1>&2
     echo "run 'gcloud config set compute/region REGION'." 1>&2
     echo "replace 'REGION' with the region name like us-west1." 1>&2
+    echo "Or provide 'REGION' as input to script as follows:" 1>&2
+    usage
     exit 1;
+  fi
 fi

hack/crdbversions/main.go

@@ -59,6 +59,7 @@ type crdbVersion struct {
 type templateData struct {
 	CrdbVersions            []crdbVersion
 	LatestStableCrdbVersion string
+	OperatorImage           string
 	OperatorVersion         string
 	GeneratedWarning        string
 	Year                    string
@@ -68,6 +69,7 @@ func main() {
 	log.SetFlags(0)
 	crdbVersionsFile := flag.String("crdb-versions", "", "YAML file with CRDB versions")
 	operatorVersion := flag.String("operator-version", "", "Current operator version")
+	operatorImage := flag.String("operator-image", "cockroachdb/cockroach-operator", "Current operator image")
 	repoRoot := flag.String("repo-root", "", "Git repository root")
 	flag.Parse()
 
@@ -87,7 +89,7 @@ func main() {
 		log.Fatalf("Cannot read versions file: %s", err)
 	}
 
-	data, err := generateTemplateData(vs, *operatorVersion)
+	data, err := generateTemplateData(vs, *operatorVersion, *operatorImage)
 	if err != nil {
 		log.Fatalf("Cannot generate template data: %s", err)
 	}
@@ -139,10 +141,11 @@ func readCrdbVersions(r io.Reader) ([]crdbVersion, error) {
 	return versions.CrdbVersions, nil
 }
 
-func generateTemplateData(crdbVersions []crdbVersion, operatorVersion string) (templateData, error) {
+func generateTemplateData(crdbVersions []crdbVersion, operatorVersion, operatorImage string) (templateData, error) {
 	var data templateData
 	data.Year = fmt.Sprint(time.Now().Year())
 	data.OperatorVersion = operatorVersion
+	data.OperatorImage = operatorImage
 	data.CrdbVersions = crdbVersions
 
 	latestStable := crdbVersions[len(crdbVersions)-1].Tag

hack/crdbversions/main_test.go

@@ -65,7 +65,7 @@ func TestGenerateTemplateData(t *testing.T) {
 		crdbVersions = append(crdbVersions, crdbVersion{Tag: r})
 	}
 
-	data, err := generateTemplateData(crdbVersions, "2.0.1")
+	data, err := generateTemplateData(crdbVersions, "2.0.1", "cockroachdb/cockroach-operator")
 	require.NoError(t, err)
 	require.Len(t, data.CrdbVersions, len(crdbVersions))
 	require.Equal(t, "1.2.3", data.LatestStableCrdbVersion)

hack/create-gke-cluster.sh

@@ -29,6 +29,8 @@ CLUSTER_NAME=""
 # TODO maybe set default zone?
 ZONE=""
 
+REGION=""
+
 # shellcheck disable=SC1090
 source "$ROOT"/common.sh
 # shellcheck disable=SC1090
@@ -39,7 +41,7 @@ enable-service compute.googleapis.com
 enable-service container.googleapis.com
 
 GKE_VERSION=$(gcloud container get-server-config \
-  --format="value(validMasterVersions[0])")
+  --format="value(validMasterVersions[0])" --region=$REGION)
 
 # Get a comma separated list of zones from the default region
 ZONESINREGION=""

hack/dev.sh

@@ -22,6 +22,7 @@ CLUSTER_NAME="dev"
 NODE_IMAGE="rancher/k3s:v1.23.3-k3s1"
 REGISTRY_NAME="registry.localhost"
 REGISTRY_PORT=5000
+DEV_REGISTRY="${REGISTRY_NAME}:${REGISTRY_PORT}"
 
 main() {
   case "${1:-}" in
@@ -45,8 +46,10 @@ install_operator() {
   # Can't seem to figure out how to leverage the stamp variables here. So for
   # now I've added a defined make variable which can be used for substitution
   # in //config/default/BUILD.bazel.
+  # ${REGISTRY_NAME} must be mapped to localhost or 127.0.0.1 to push the image to the k3d registry.
+  bazel run //hack/crdbversions:crdbversions -- -operator-image ${DEV_REGISTRY}/cockroach-operator -operator-version ${APP_VERSION} -crdb-versions $(PWD)/crdb-versions.yaml -repo-root $(PWD)
   K8S_CLUSTER="k3d-${CLUSTER_NAME}" \
-    DEV_REGISTRY="${REGISTRY_NAME}:${REGISTRY_PORT}" \
+    DEV_REGISTRY="${DEV_REGISTRY}" \
     bazel run \
     --stamp \
     --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 \