Update to Kubernetes 1.24.12 + Use Go 1.19 (#44)

This PR is a continuation of the work in #32 and contains the following changes:

Use Go 1.19
Update Kubernetes deps to 1.24.12
Move the cloudstack-go dep to its new home in the apache org (and update to v2.15.0)
switch to klog v2
I can provide a test container image if needed.

Signed-off-by: Hans Rakers <[email protected]>
Co-authored-by: DESKTOP-AG9326G\Schlatter <[email protected]>

Author: hrak

.github/workflows/build.yml

@@ -33,7 +33,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v3
       with:
-        go-version: 1.13
+        go-version: 1.19
     - name: Run Script
       run:  make test
 

Dockerfile

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-FROM golang:1.13 as builder
+FROM golang:1.19 as builder
 COPY . /go/src/github.com/apache/cloudstack-kubernetes-provider
 WORKDIR /go/src/github.com/apache/cloudstack-kubernetes-provider
 RUN make clean && CGO_ENABLED=0 GOOS=linux make

cloudstack.go

@@ -26,11 +26,11 @@ import (
 	"io"
 	"os"
 
-	"github.com/xanzy/go-cloudstack/v2/cloudstack"
+	"github.com/apache/cloudstack-go/v2/cloudstack"
 	"gopkg.in/gcfg.v1"
 	"k8s.io/apimachinery/pkg/types"
 	cloudprovider "k8s.io/cloud-provider"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // ProviderName is the name of this cloud provider.
@@ -120,6 +120,14 @@ func (cs *CSCloud) Instances() (cloudprovider.Instances, bool) {
 	return cs, true
 }
 
+func (cs *CSCloud) InstancesV2() (cloudprovider.InstancesV2, bool) {
+	if cs.client == nil {
+		return nil, false
+	}
+
+	return cs, true
+}
+
 // Zones returns an implementation of Zones for CloudStack.
 func (cs *CSCloud) Zones() (cloudprovider.Zones, bool) {
 	if cs.client == nil {

cloudstack_instances.go

@@ -25,17 +25,17 @@ import (
 	"fmt"
 	"regexp"
 
-	"github.com/xanzy/go-cloudstack/v2/cloudstack"
-	"k8s.io/api/core/v1"
+	"github.com/apache/cloudstack-go/v2/cloudstack"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	cloudprovider "k8s.io/cloud-provider"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 var labelInvalidCharsRegex *regexp.Regexp = regexp.MustCompile(`([^A-Za-z0-9][^-A-Za-z0-9_.]*)?[^A-Za-z0-9]`)
 
 // NodeAddresses returns the addresses of the specified instance.
-func (cs *CSCloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.NodeAddress, error) {
+func (cs *CSCloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]corev1.NodeAddress, error) {
 	instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByName(
 		string(name),
 		cloudstack.WithProject(cs.projectID),
@@ -51,7 +51,7 @@ func (cs *CSCloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1
 }
 
 // NodeAddressesByProviderID returns the addresses of the specified instance.
-func (cs *CSCloud) NodeAddressesByProviderID(ctx context.Context, providerID string) ([]v1.NodeAddress, error) {
+func (cs *CSCloud) NodeAddressesByProviderID(ctx context.Context, providerID string) ([]corev1.NodeAddress, error) {
 	instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByID(
 		providerID,
 		cloudstack.WithProject(cs.projectID),
@@ -66,21 +66,21 @@ func (cs *CSCloud) NodeAddressesByProviderID(ctx context.Context, providerID str
 	return cs.nodeAddresses(instance)
 }
 
-func (cs *CSCloud) nodeAddresses(instance *cloudstack.VirtualMachine) ([]v1.NodeAddress, error) {
+func (cs *CSCloud) nodeAddresses(instance *cloudstack.VirtualMachine) ([]corev1.NodeAddress, error) {
 	if len(instance.Nic) == 0 {
 		return nil, errors.New("instance does not have an internal IP")
 	}
 
-	addresses := []v1.NodeAddress{
-		{Type: v1.NodeInternalIP, Address: instance.Nic[0].Ipaddress},
+	addresses := []corev1.NodeAddress{
+		{Type: corev1.NodeInternalIP, Address: instance.Nic[0].Ipaddress},
 	}
 
 	if instance.Hostname != "" {
-		addresses = append(addresses, v1.NodeAddress{Type: v1.NodeHostName, Address: instance.Hostname})
+		addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeHostName, Address: instance.Hostname})
 	}
 
 	if instance.Publicip != "" {
-		addresses = append(addresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: instance.Publicip})
+		addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: instance.Publicip})
 	} else {
 		// Since there is no sane way to determine the external IP if the host isn't
 		// using static NAT, we will just fire a log message and omit the external IP.
@@ -168,3 +168,43 @@ func (cs *CSCloud) InstanceExistsByProviderID(ctx context.Context, providerID st
 func (cs *CSCloud) InstanceShutdownByProviderID(ctx context.Context, providerID string) (bool, error) {
 	return false, cloudprovider.NotImplemented
 }
+
+func (cs *CSCloud) InstanceExists(ctx context.Context, node *corev1.Node) (bool, error) {
+	nodeName := types.NodeName(node.Name)
+	providerID, err := cs.InstanceID(ctx, nodeName)
+	if err != nil {
+		return false, err
+	}
+
+	return cs.InstanceExistsByProviderID(ctx, providerID)
+}
+
+func (cs *CSCloud) InstanceShutdown(ctx context.Context, node *corev1.Node) (bool, error) {
+	return false, cloudprovider.NotImplemented
+}
+
+func (cs *CSCloud) InstanceMetadata(ctx context.Context, node *corev1.Node) (*cloudprovider.InstanceMetadata, error) {
+
+	instanceType, err := cs.InstanceType(ctx, types.NodeName(node.Name))
+	if err != nil {
+		return nil, err
+	}
+
+	addresses, err := cs.NodeAddresses(ctx, types.NodeName(node.Name))
+	if err != nil {
+		return nil, err
+	}
+
+	zone, err := cs.GetZone(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return &cloudprovider.InstanceMetadata{
+		ProviderID:    cs.ProviderName(),
+		InstanceType:  instanceType,
+		NodeAddresses: addresses,
+		Zone:          cs.zone,
+		Region:        zone.Region,
+	}, nil
+}

cloudstack_loadbalancer.go

@@ -25,10 +25,10 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/xanzy/go-cloudstack/v2/cloudstack"
-	"k8s.io/klog"
+	"github.com/apache/cloudstack-go/v2/cloudstack"
+	"k8s.io/klog/v2"
 
-	v1 "k8s.io/api/core/v1"
+	corev1 "k8s.io/api/core/v1"
 	cloudprovider "k8s.io/cloud-provider"
 )
 
@@ -50,7 +50,7 @@ type loadBalancer struct {
 }
 
 // GetLoadBalancer returns whether the specified load balancer exists, and if so, what its status is.
-func (cs *CSCloud) GetLoadBalancer(ctx context.Context, clusterName string, service *v1.Service) (*v1.LoadBalancerStatus, bool, error) {
+func (cs *CSCloud) GetLoadBalancer(ctx context.Context, clusterName string, service *corev1.Service) (*corev1.LoadBalancerStatus, bool, error) {
 	klog.V(4).Infof("GetLoadBalancer(%v, %v, %v)", clusterName, service.Namespace, service.Name)
 
 	// Get the load balancer details and existing rules.
@@ -66,14 +66,14 @@ func (cs *CSCloud) GetLoadBalancer(ctx context.Context, clusterName string, serv
 
 	klog.V(4).Infof("Found a load balancer associated with IP %v", lb.ipAddr)
 
-	status := &v1.LoadBalancerStatus{}
-	status.Ingress = append(status.Ingress, v1.LoadBalancerIngress{IP: lb.ipAddr})
+	status := &corev1.LoadBalancerStatus{}
+	status.Ingress = append(status.Ingress, corev1.LoadBalancerIngress{IP: lb.ipAddr})
 
 	return status, true, nil
 }
 
 // EnsureLoadBalancer creates a new load balancer, or updates the existing one. Returns the status of the balancer.
-func (cs *CSCloud) EnsureLoadBalancer(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node) (status *v1.LoadBalancerStatus, err error) {
+func (cs *CSCloud) EnsureLoadBalancer(ctx context.Context, clusterName string, service *corev1.Service, nodes []*corev1.Node) (status *corev1.LoadBalancerStatus, err error) {
 	klog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v)", clusterName, service.Namespace, service.Name, service.Spec.LoadBalancerIP, service.Spec.Ports, nodes)
 
 	if len(service.Spec.Ports) == 0 {
@@ -88,9 +88,9 @@ func (cs *CSCloud) EnsureLoadBalancer(ctx context.Context, clusterName string, s
 
 	// Set the load balancer algorithm.
 	switch service.Spec.SessionAffinity {
-	case v1.ServiceAffinityNone:
+	case corev1.ServiceAffinityNone:
 		lb.algorithm = "roundrobin"
-	case v1.ServiceAffinityClientIP:
+	case corev1.ServiceAffinityClientIP:
 		lb.algorithm = "source"
 	default:
 		return nil, fmt.Errorf("unsupported load balancer affinity: %v", service.Spec.SessionAffinity)
@@ -193,14 +193,14 @@ func (cs *CSCloud) EnsureLoadBalancer(ctx context.Context, clusterName string, s
 		}
 	}
 
-	status = &v1.LoadBalancerStatus{}
-	status.Ingress = []v1.LoadBalancerIngress{{IP: lb.ipAddr}}
+	status = &corev1.LoadBalancerStatus{}
+	status.Ingress = []corev1.LoadBalancerIngress{{IP: lb.ipAddr}}
 
 	return status, nil
 }
 
 // UpdateLoadBalancer updates hosts under the specified load balancer.
-func (cs *CSCloud) UpdateLoadBalancer(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node) error {
+func (cs *CSCloud) UpdateLoadBalancer(ctx context.Context, clusterName string, service *corev1.Service, nodes []*corev1.Node) error {
 	klog.V(4).Infof("UpdateLoadBalancer(%v, %v, %v, %v)", clusterName, service.Namespace, service.Name, nodes)
 
 	// Get the load balancer details and existing rules.
@@ -246,7 +246,7 @@ func (cs *CSCloud) UpdateLoadBalancer(ctx context.Context, clusterName string, s
 
 // EnsureLoadBalancerDeleted deletes the specified load balancer if it exists, returning
 // nil if the load balancer specified either didn't exist or was successfully deleted.
-func (cs *CSCloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName string, service *v1.Service) error {
+func (cs *CSCloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName string, service *corev1.Service) error {
 	klog.V(4).Infof("EnsureLoadBalancerDeleted(%v, %v, %v)", clusterName, service.Namespace, service.Name)
 
 	// Get the load balancer details and existing rules.
@@ -286,12 +286,12 @@ func (cs *CSCloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName st
 }
 
 // GetLoadBalancerName retrieves the name of the LoadBalancer.
-func (cs *CSCloud) GetLoadBalancerName(ctx context.Context, clusterName string, service *v1.Service) string {
+func (cs *CSCloud) GetLoadBalancerName(ctx context.Context, clusterName string, service *corev1.Service) string {
 	return cloudprovider.DefaultLoadBalancerName(service)
 }
 
 // getLoadBalancer retrieves the IP address and ID and all the existing rules it can find.
-func (cs *CSCloud) getLoadBalancer(service *v1.Service) (*loadBalancer, error) {
+func (cs *CSCloud) getLoadBalancer(service *corev1.Service) (*loadBalancer, error) {
 	lb := &loadBalancer{
 		CloudStackClient: cs.client,
 		name:             cs.GetLoadBalancerName(context.TODO(), "", service),
@@ -329,7 +329,7 @@ func (cs *CSCloud) getLoadBalancer(service *v1.Service) (*loadBalancer, error) {
 }
 
 // verifyHosts verifies if all hosts belong to the same network, and returns the host ID's and network ID.
-func (cs *CSCloud) verifyHosts(nodes []*v1.Node) ([]string, string, error) {
+func (cs *CSCloud) verifyHosts(nodes []*corev1.Node) ([]string, string, error) {
 	hostNames := map[string]bool{}
 	for _, node := range nodes {
 		hostNames[strings.ToLower(node.Name)] = true
@@ -456,7 +456,7 @@ func (lb *loadBalancer) releaseLoadBalancerIP() error {
 
 // checkLoadBalancerRule checks if the rule already exists and if it does, if it can be updated. If
 // it does exist but cannot be updated, it will delete the existing rule so it can be created again.
-func (lb *loadBalancer) checkLoadBalancerRule(lbRuleName string, port v1.ServicePort, protocol LoadBalancerProtocol) (*cloudstack.LoadBalancerRule, bool, error) {
+func (lb *loadBalancer) checkLoadBalancerRule(lbRuleName string, port corev1.ServicePort, protocol LoadBalancerProtocol) (*cloudstack.LoadBalancerRule, bool, error) {
 	lbRule, ok := lb.rules[lbRuleName]
 	if !ok {
 		return nil, false, nil
@@ -490,7 +490,7 @@ func (lb *loadBalancer) updateLoadBalancerRule(lbRuleName string, protocol LoadB
 }
 
 // createLoadBalancerRule creates a new load balancer rule and returns it's ID.
-func (lb *loadBalancer) createLoadBalancerRule(lbRuleName string, port v1.ServicePort, protocol LoadBalancerProtocol) (*cloudstack.LoadBalancerRule, error) {
+func (lb *loadBalancer) createLoadBalancerRule(lbRuleName string, port corev1.ServicePort, protocol LoadBalancerProtocol) (*cloudstack.LoadBalancerRule, error) {
 	p := lb.LoadBalancer.NewCreateLoadBalancerRuleParams(
 		lb.algorithm,
 		lbRuleName,

cloudstack_test.go

@@ -26,7 +26,7 @@ import (
 	"strings"
 	"testing"
 
-	"k8s.io/api/core/v1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -111,7 +111,7 @@ func TestLoadBalancer(t *testing.T) {
 		t.Fatalf("LoadBalancer() returned false")
 	}
 
-	_, exists, err := lb.GetLoadBalancer(context.TODO(), testClusterName, &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "noexist"}})
+	_, exists, err := lb.GetLoadBalancer(context.TODO(), testClusterName, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "noexist"}})
 	if err != nil {
 		t.Fatalf("GetLoadBalancer(\"noexist\") returned error: %s", err)
 	}

cmd/cloudstack-ccm/main.go

@@ -24,14 +24,18 @@ import (
 	"math/rand"
 	"time"
 
-	"k8s.io/klog"
+	"k8s.io/apimachinery/pkg/util/wait"
+	cloudprovider "k8s.io/cloud-provider"
 
+	"k8s.io/cloud-provider/app"
+	"k8s.io/cloud-provider/app/config"
+	"k8s.io/cloud-provider/options"
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/component-base/logs"
-	"k8s.io/kubernetes/cmd/cloud-controller-manager/app"
 
 	_ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugins
 	_ "k8s.io/component-base/metrics/prometheus/version"  // for version metric registration
+	"k8s.io/klog/v2"
 
 	"github.com/spf13/pflag"
 
@@ -41,7 +45,14 @@ import (
 func main() {
 	rand.Seed(time.Now().UnixNano())
 
-	command := app.NewCloudControllerManagerCommand()
+	ccmOptions, err := options.NewCloudControllerManagerOptions()
+	if err != nil {
+		klog.Fatalf("unable to initialize command options: %v", err)
+	}
+
+	fss := cliflag.NamedFlagSets{}
+
+	command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, app.DefaultInitFuncConstructors, fss, wait.NeverStop)
 
 	// TODO: once we switch everything over to Cobra commands, we can go back to calling
 	// cliflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
@@ -56,3 +67,25 @@ func main() {
 		klog.Fatalf("error: %v\n", err)
 	}
 }
+
+func cloudInitializer(config *config.CompletedConfig) cloudprovider.Interface {
+	cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider
+
+	// initialize cloud provider with the cloud provider name and config file provided
+	cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile)
+	if err != nil {
+		klog.Fatalf("Cloud provider could not be initialized: %v", err)
+	}
+	if cloud == nil {
+		klog.Fatalf("Cloud provider is nil")
+	}
+
+	if !cloud.HasClusterID() {
+		if config.ComponentConfig.KubeCloudShared.AllowUntaggedCloud {
+			klog.Warning("detected a cluster without a ClusterID.  A ClusterID will be required in the future.  Please tag your cluster to avoid any future issues")
+		} else {
+			klog.Fatalf("no ClusterID found.  A ClusterID is required for the cloud provider to function properly.  This check can be bypassed by setting the allow-untagged-cloud option")
+		}
+	}
+	return cloud
+}