NLB and Block Volume Performance Levels support release to Open Source from release-1.19 branch (#379)

* Add support for NLB lifecycle management in CCM

* JIRA: OKE-16175 - Add support for Block Volume Performance Levels with CSI

* JIRA: OKE-20067 Bug fix: CSI Expand Volume - PV PVC sync error for requested volume size less than or equal to old size

* task/OKE-16737 Made change to not remove nodes from LB backend set if all nodes are not ready

* JIRA: OKE-20459 - CSI node driver should work even if vpusPerGB is not set

* Open Source Release from branch release-1.19 (nlb, oci-sdk v50)

Co-authored-by: Akarsh Ellore Sreenath <[email protected]>
Co-authored-by: mpagnis <[email protected]>
Co-authored-by: yashg <[email protected]>
Co-authored-by: Pranav Sriram <[email protected]>

Author: 5 people

cmd/oci-csi-controller-driver/csi-controller-driver/oci-csi-controller-driver.go

@@ -22,12 +22,12 @@ import (
 )
 
 //StartControllerDriver main function to start CSI Controller Driver
-func StartControllerDriver(csiOptions csioptions.CSIOptions) {
+func StartControllerDriver(csioptions csioptions.CSIOptions) {
 
 	logger := logging.Logger().Sugar()
 	logger.Sync()
 
-	drv, err := driver.NewControllerDriver(logger.Named("BV"), csiOptions.Endpoint, csiOptions.Kubeconfig, csiOptions.Master,
+	drv, err := driver.NewControllerDriver(logger.Named("BV").With(zap.String("component", "csi-controller")), csioptions.Endpoint, csioptions.Kubeconfig, csioptions.Master,
 		true, driver.BlockVolumeDriverName, driver.BlockVolumeDriverVersion)
 	if err != nil {
 		logger.With(zap.Error(err)).Fatal("Failed to create controller driver.")

cmd/oci-flexvolume-driver/main.go

@@ -47,6 +47,7 @@ func main() {
 		"pid", os.Getpid(),
 		"version", version,
 		"build", build,
+		"component", "flexvolume-driver",
 	)
 	logger.Debug("OCI Flexvolume driver")
 	d, err := block.NewOCIFlexvolumeDriver(logger)

cmd/oci-volume-provisioner/main.go

@@ -43,7 +43,7 @@ func main() {
 
 	logger := log.Sugar()
 
-	logger.With("version", version, "build", build).Info("oci-volume-provisioner")
+	logger.With("version", version, "build", build, "component", "volume-provisioner").Info("oci-volume-provisioner")
 
 	// Set up signals so we handle the shutdown signal gracefully.
 	stopCh := signals.SetupSignalHandler()

go.mod

@@ -40,7 +40,7 @@ require (
 	github.com/kubernetes-csi/csi-lib-utils v0.8.1
 	github.com/onsi/ginkgo v1.14.1
 	github.com/onsi/gomega v1.10.2
-	github.com/oracle/oci-go-sdk/v31 v31.0.0
+	github.com/oracle/oci-go-sdk/v50 v50.1.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.9.0
 	github.com/spf13/pflag v1.0.5

go.sum

@@ -471,6 +471,10 @@ github.com/opencontainers/selinux v1.5.1/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwy
 github.com/opencontainers/selinux v1.5.2/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwyzAJufJyiTt7s0g=
 github.com/oracle/oci-go-sdk/v31 v31.0.0 h1:LEstCAlSNcuuPIRUMLuX3UF+V3ID2B1LN4k9Dvbd5tM=
 github.com/oracle/oci-go-sdk/v31 v31.0.0/go.mod h1:4MlbGGDlqigjplvUfbVecEqNCN2cYZbdJ8PpAMVAQkw=
+github.com/oracle/oci-go-sdk/v49 v49.2.0 h1:l4PUk81EKdTDD4mDg5wrELpdWFqYeE9KYejfNgtsyUI=
+github.com/oracle/oci-go-sdk/v49 v49.2.0/go.mod h1:E8q2DXmXnSozLdXHUFF+o3L2gzcWbiFIPFYOYWdqOfc=
+github.com/oracle/oci-go-sdk/v50 v50.1.0 h1:6WyLNxEmnUuc1fYYVTuIeH2iQiGVdwlE//qSuYF7bK4=
+github.com/oracle/oci-go-sdk/v50 v50.1.0/go.mod h1:5tboHcoGIrAUerGgH3Sdc6eJdhw5YyywE78g99hzHpw=
 github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
@@ -520,6 +524,8 @@ github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIK
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/sony/gobreaker v0.4.2-0.20210216022020-dd874f9dd33b h1:br+bPNZsJWKicw/5rALEo67QHs5weyD5tf8WST+4sJ0=
+github.com/sony/gobreaker v0.4.2-0.20210216022020-dd874f9dd33b/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=

hack/existing-standalone-cluster-env-template.sh

@@ -16,6 +16,17 @@
 # The test suites to run (can replace or add tags)
 export FOCUS="\[cloudprovider\]"
 
+# Scope can be ARM / AMD / BOTH
+# Mandatory
+export SCOPE="BOTH"
+
+# A Reserved IP in your compartment for testing LB creation with Reserved IP
+# Create a public reserved IP in your compartment using the following link:
+# https://docs.oracle.com/en-us/iaas/Content/Network/Tasks/managingpublicIPs.htm#console-reserved
+# Set the public reserved IP in the following env-variable:
+# Mandatory
+export RESERVED_IP=""
+
 # Set path to kubeconfig of existing cluster if it does not exist in default path. Defaults to $HOME/.kube/config.
 # Mandatory
 export CLUSTER_KUBECONFIG=$HOME/.kube/config
@@ -39,13 +50,19 @@ export CMEK_KMS_KEY=""
 # Mandatory
 export NSG_OCIDS=""
 
-# VOLUME HANDLE in the format filesystem_ocid:mountTargetIP:export_path
-# make sure volume handle is in the same subnet as your nodes
-export VOLUME_HANDLE=""
-# OPTIONAL
+# FSS VOLUME HANDLE in the format filesystem_ocid:mountTargetIP:export_path
+# Make sure fss volume handle is in the same subnet as your nodes
+# Create a file system, file export path and mount target in your VCN by following
+# https://docs.oracle.com/en-us/iaas/Content/File/Tasks/creatingfilesystems.htm#Using_the_Console
+# And setup your network for the file system by following:
+# https://docs.oracle.com/en-us/iaas/Content/File/Tasks/securitylistsfilestorage.htm
+# Mandatory
+export FSS_VOLUME_HANDLE=""
 
 # For debugging the tests in existing cluster, do not turn it off by default.
+# Optional
 # export DELETE_NAMESPACE=false
 
 # By default, public images are used. But if your Cluster's environment cannot access above public images then below option can be used to specify an accessible repo.
+# Optional
 # export IMAGE_PULL_REPO="accessiblerepo.com/repo/path/"

hack/run_e2e_test.sh

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-function check-env {
+function check-env () {
     if [ -z "$2" ]; then
         echo "$1 must be defined"
         exit 1
@@ -27,11 +27,13 @@ check-env "ADLOCATION"      $ADLOCATION
 check-env "NSG_OCIDS"       $NSG_OCIDS
 check-env "VOLUME_HANDLE"       $VOLUME_HANDLE
 
-if [ -z "$IMAGE_PULL_REPO" ]; then
-    IMAGE_PULL_REPO=""
-fi
+function set_image_pull_repo_and_delete_namespace_flag () {
+    if [ -z "$IMAGE_PULL_REPO" ]; then
+        IMAGE_PULL_REPO=""
+    fi
 
-DELETE_NAMESPACE=${DELETE_NAMESPACE:-"true"}
+    DELETE_NAMESPACE=${DELETE_NAMESPACE:-"true"}
+}
 
 function run_e2e_tests_existing_cluster() {
     ginkgo -v -progress --trace "${FOCUS_OPT}" "${FOCUS_FP_OPT}"  \
@@ -44,42 +46,99 @@ function run_e2e_tests_existing_cluster() {
         --cmek-kms-key=${CMEK_KMS_KEY} \
         --nsg-ocids=${NSG_OCIDS} \
         --reserved-ip=${RESERVED_IP} \
-        --volume-handle=${VOLUME_HANDLE} \
+        --architecture=${ARCHITECTURE} \
+        --volume-handle=${FSS_VOLUME_HANDLE}
     retval=$?
-    exit $retval
+    return $retval
 }
 
-# The FOCUS environment variable can be set with a regex to tun selected tests
-# e.g. export FOCUS="\[cloudprovider\]"
-export FOCUS_OPT=""
-export FOCUS_FP_OPT=""
-if [ ! -z "${FOCUS}" ]; then
-    # Because we tag our test descriptions with tags that are surrounded
-    # by square brackets, we have to escape the brackets when we set the
-    # FOCUS variable to match on a bracket rather than have it interpreted
-    # as a regex character class. The check below looks to see if the FOCUS
-    # has square brackets which aren't yet escaped and fixes them if needed.
-    re1='^\[.+\]$' # [ccm]
-    if [[ "${FOCUS}" =~ $re1 ]]; then
-        echo -E "Escaping square brackes in ${FOCUS} to work as a regex match."
-        FOCUS=$(echo $FOCUS|sed -e 's/\[/\\[/g' -e 's/\]/\\]/g')
-        echo -E "Modified FOCUS value to: ${FOCUS}"
-    fi
+function set_focus () {
+    # The FOCUS environment variable can be set with a regex to tun selected tests
+    # e.g. export FOCUS="\[cloudprovider\]"
+    export FOCUS_OPT=""
+    export FOCUS_FP_OPT=""
+    if [ ! -z "${FOCUS}" ]; then
+        # Because we tag our test descriptions with tags that are surrounded
+        # by square brackets, we have to escape the brackets when we set the
+        # FOCUS variable to match on a bracket rather than have it interpreted
+        # as a regex character class. The check below looks to see if the FOCUS
+        # has square brackets which aren't yet escaped and fixes them if needed.
+        re1='^\[.+\]$' # [ccm]
+        if [[ "${FOCUS}" =~ $re1 ]]; then
+            echo -E "Escaping square brackes in ${FOCUS} to work as a regex match."
+            FOCUS=$(echo $FOCUS|sed -e 's/\[/\\[/g' -e 's/\]/\\]/g')
+            echo -E "Modified FOCUS value to: ${FOCUS}"
+        fi
 
-    echo "Running focused tests: ${FOCUS}"
-    FOCUS_OPT="-focus=${FOCUS}"
+        echo "Running focused tests: ${FOCUS}"
+        FOCUS_OPT="-focus=${FOCUS}"
 
-    # The FILES environment variable can be defined to interpret the regex as a
-    # set of files.
-    # e.g. export FILES="true"
-    if [[ ! -z "${FILES}" && "${FILES}" == "true" ]]; then
-        echo "Running focused test regex as filepath expression."
-        FOCUS_FP_OPT="-regexScansFilePath=${FILES}"
+        # The FILES environment variable can be defined to interpret the regex as a
+        # set of files.
+        # e.g. export FILES="true"
+        if [[ ! -z "${FILES}" && "${FILES}" == "true" ]]; then
+            echo "Running focused test regex as filepath expression."
+            FOCUS_FP_OPT="-regexScansFilePath=${FILES}"
+        fi
     fi
-fi
+}
 
 echo "CLUSTER_KUBECONFIG is ${CLUSTER_KUBECONFIG}"
 echo "CLOUD_CONFIG is ${CLOUD_CONFIG}"
 
-# run the ginko test framework for existing cluster
-run_e2e_tests_existing_cluster
+function run_tests () {
+    set_image_pull_repo_and_delete_namespace_flag
+    set_focus
+    # run the ginko test framework for existing cluster
+    # run ARM tests
+    if [[ "$SCOPE" == "BOTH" || "$SCOPE" == "ARM" ]]; then
+        run_e2e_tests_existing_cluster
+        retval_arm=$?
+    fi
+    # run AMD tests
+    if [[ "$SCOPE" == "BOTH" || "$SCOPE" == "AMD" ]]; then
+        run_e2e_tests_existing_cluster
+        retval_amd=$?
+    fi
+
+    RED='\033[0;31m'
+    NC='\033[0m' # No Color
+    if [[ "$SCOPE" == "BOTH" ]]; then
+        if [[ $retval_amd == 0 && $retval_arm == 0 ]]; then
+            printf "ARM and AMD tests are Successful!"
+            return $retval_amd
+        fi
+
+        if [[ $retval_amd != 0 ]]; then
+            printf "${RED}AMD Failed${NC}"
+            return $retval_amd
+        fi
+
+        if [[ $retval_arm != 0 ]]; then
+            printf "${RED}ARM Failed${NC}"
+            return $retval_arm
+        fi
+    fi
+
+    if [[ "$SCOPE" == "ARM" ]]; then
+        if [[ $retval_arm != 0 ]]; then
+            printf "${RED}ARM Failed${NC}"
+            return $retval_arm
+        else
+            echo "ARM tests are Successful"
+            return $retval_arm
+        fi
+    fi
+
+    if [[ "$SCOPE" == "AMD" ]]; then
+        if [[ $retval_amd != 0 ]]; then
+            printf "${RED}AMD Failed${NC}"
+            return $retval_amd
+        else
+            echo "AMD tests are Successful"
+            return $retval_amd
+        fi
+    fi
+}
+
+run_tests

pkg/cloudprovider/providers/oci/ccm.go

@@ -36,8 +36,8 @@ import (
 	"github.com/oracle/oci-cloud-controller-manager/pkg/metrics"
 	"github.com/oracle/oci-cloud-controller-manager/pkg/oci/client"
 	"github.com/oracle/oci-cloud-controller-manager/pkg/oci/instance/metadata"
-	"github.com/oracle/oci-go-sdk/v31/common"
-	"github.com/oracle/oci-go-sdk/v31/core"
+	"github.com/oracle/oci-go-sdk/v50/common"
+	"github.com/oracle/oci-go-sdk/v50/core"
 )
 
 const (

pkg/cloudprovider/providers/oci/config/config.go

@@ -19,8 +19,8 @@ import (
 	"io"
 	"os"
 
-	"github.com/oracle/oci-go-sdk/v31/common"
-	"github.com/oracle/oci-go-sdk/v31/common/auth"
+	"github.com/oracle/oci-go-sdk/v50/common"
+	"github.com/oracle/oci-go-sdk/v50/common/auth"
 	"github.com/pkg/errors"
 
 	"go.uber.org/zap"

pkg/cloudprovider/providers/oci/instances.go

@@ -19,7 +19,7 @@ import (
 	"fmt"
 	"net"
 
-	"github.com/oracle/oci-go-sdk/v31/core"
+	"github.com/oracle/oci-go-sdk/v50/core"
 	"k8s.io/apimachinery/pkg/labels"
 
 	"github.com/oracle/oci-cloud-controller-manager/pkg/oci/client"