Skip to content

Commit

Permalink
Merge pull request #296 from gujingit/dev/e2e-test
Browse files Browse the repository at this point in the history 
add e2etests for new annotations
  • Loading branch information
@gujingit
gujingit authored Mar 17, 2022
2 parents e23e21d + ca10c05 commit 21173d2
Show file tree
Hide file tree
Showing 8 changed files with 269 additions and 39 deletions.
65 changes: 62 additions & 3 deletions e2e/framework/expect.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,16 @@ func loadBalancerAttrEqual(f *Framework, anno *service.AnnotationRequest, svc *v
return fmt.Errorf("expected slb networkType %s, got %s", networkType, lb.NetworkType)
}
}

if hostName := anno.Get(service.HostName); hostName != "" {
if len(svc.Status.LoadBalancer.Ingress) != 1 ||
svc.Status.LoadBalancer.Ingress[0].Hostname != hostName ||
svc.Status.LoadBalancer.Ingress[0].IP != "" {
return fmt.Errorf("svc ingress hostname %v is not equal to hostname %s",
svc.Status.LoadBalancer.Ingress, hostName)
}
}

if eipAnno := anno.Get(service.ExternalIPType); eipAnno == "eip" {
eips, err := f.Client.CloudClient.DescribeEipAddresses(context.TODO(), string(vpc.SlbInstance), lb.LoadBalancerId)
if err != nil {
Expand All @@ -208,12 +218,22 @@ func loadBalancerAttrEqual(f *Framework, anno *service.AnnotationRequest, svc *v
return fmt.Errorf("lb %s has %d eips", lb.LoadBalancerId, len(eips))
}

if len(svc.Status.LoadBalancer.Ingress) != 1 ||
svc.Status.LoadBalancer.Ingress[0].IP != eips[0] {
// hostname annotation takes effect first.
// if set hostname annotation, ip should be nil
if anno.Get(service.HostName) == "" &&
(len(svc.Status.LoadBalancer.Ingress) != 1 ||
svc.Status.LoadBalancer.Ingress[0].IP != eips[0]) {
return fmt.Errorf("svc ingress ip %v is not equal to eip %s",
svc.Status.LoadBalancer.Ingress, eips[0])
}
}

if anno.Get(service.ExternalIPType) == "" && anno.Get(service.HostName) == "" {
if len(svc.Status.LoadBalancer.Ingress) != 1 ||
svc.Status.LoadBalancer.Ingress[0].IP != lb.Address {
return fmt.Errorf("svc ingress ip %v is not equal to slb ip %s",
svc.Status.LoadBalancer.Ingress, lb.Address)
}
}
return nil
}
Expand Down Expand Up @@ -402,6 +422,15 @@ func tcpEqual(reqCtx *service.RequestContext, local v1.ServicePort, remote model
return fmt.Errorf("expected slb persistenceTimeout %d, got %d", timeout, remote.PersistenceTimeout)
}
}
if establishedTimeout := reqCtx.Anno.Get(service.EstablishedTimeout); establishedTimeout != "" {
timeout, err := strconv.Atoi(establishedTimeout)
if err != nil {
return fmt.Errorf("establishedTimeout %s parse error: %s", establishedTimeout, err.Error())
}
if remote.EstablishedTimeout != timeout {
return fmt.Errorf("expected slb establishedTimeout %d, got %d", timeout, remote.EstablishedTimeout)
}
}
if healthCheckConnectTimeout := reqCtx.Anno.Get(service.HealthCheckConnectTimeout); healthCheckConnectTimeout != "" {
timeout, err := strconv.Atoi(healthCheckConnectTimeout)
if err != nil {
Expand Down Expand Up @@ -519,6 +548,11 @@ func httpEqual(reqCtx *service.RequestContext, local v1.ServicePort, remote mode
return fmt.Errorf("expected slb healthCheckTimeout %d, got %d", timeout, remote.HealthCheckTimeout)
}
}
if healthCheckMethod := reqCtx.Anno.Get(service.HealthCheckMethod); healthCheckMethod != "" {
if remote.HealthCheckMethod != healthCheckMethod {
return fmt.Errorf("expected slb healthCheckMethod %s, got %s", healthCheckMethod, remote.HealthCheckMethod)
}
}
}
}

Expand Down Expand Up @@ -555,6 +589,15 @@ func httpEqual(reqCtx *service.RequestContext, local v1.ServicePort, remote mode
return fmt.Errorf("expected slb XForwardedForProto %s, got %s", xForwardedForProto, remote.XForwardedForProto)
}
}
if requestTimeout := reqCtx.Anno.Get(service.RequestTimeout); requestTimeout != "" {
timeout, err := strconv.Atoi(requestTimeout)
if err != nil {
return fmt.Errorf("requestTimeout %s parse error: %s", requestTimeout, err.Error())
}
if remote.RequestTimeout != timeout {
return fmt.Errorf("expected slb requestTimeout %d, got %d", timeout, remote.RequestTimeout)
}
}
if idleTimeout := reqCtx.Anno.Get(service.IdleTimeout); idleTimeout != "" {
timeout, err := strconv.Atoi(idleTimeout)
if err != nil {
Expand Down Expand Up @@ -616,6 +659,11 @@ func httpsEqual(reqCtx *service.RequestContext, local v1.ServicePort, remote mod
return fmt.Errorf("expected slb healthCheckTimeout %d, got %d", timeout, remote.HealthCheckTimeout)
}
}
if healthCheckMethod := reqCtx.Anno.Get(service.HealthCheckMethod); healthCheckMethod != "" {
if remote.HealthCheckMethod != healthCheckMethod {
return fmt.Errorf("expected slb healthCheckMethod %s, got %s", healthCheckMethod, remote.HealthCheckMethod)
}
}
}
}

Expand Down Expand Up @@ -675,6 +723,15 @@ func httpsEqual(reqCtx *service.RequestContext, local v1.ServicePort, remote mod
return fmt.Errorf("expected slb enableHttp2 %s, got %s", enableHttp2, remote.EnableHttp2)
}
}
if requestTimeout := reqCtx.Anno.Get(service.RequestTimeout); requestTimeout != "" {
timeout, err := strconv.Atoi(requestTimeout)
if err != nil {
return fmt.Errorf("requestTimeout %s parse error: %s", requestTimeout, err.Error())
}
if remote.RequestTimeout != timeout {
return fmt.Errorf("expected slb requestTimeout %d, got %d", timeout, remote.RequestTimeout)
}
}
return nil
}

Expand Down Expand Up @@ -1297,7 +1354,9 @@ func (f *Framework) FindLoadBalancer() (*v1.Service, *model.LoadBalancer, error)
if err != nil {
return false, nil
}
if len(svc.Status.LoadBalancer.Ingress) == 1 && svc.Status.LoadBalancer.Ingress[0].IP != "" {
if len(svc.Status.LoadBalancer.Ingress) == 1 &&
(svc.Status.LoadBalancer.Ingress[0].IP != "" ||
svc.Status.LoadBalancer.Ingress[0].Hostname != "") {
return true, nil
}
return false, nil
Expand Down
45 changes: 34 additions & 11 deletions e2e/framework/framework.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,12 +110,16 @@ func (f *Framework) CreateCloudResource() error {
AddressType: model.InternetAddressType,
LoadBalancerSpec: model.S1Small,
RegionId: region,
LoadBalancerName: client.Namespace + "-internet-slb",
LoadBalancerName: fmt.Sprintf("%s-%s-slb", options.TestConfig.ClusterId, "internet"),
},
}
err := f.Client.CloudClient.CreateLoadBalancer(context.TODO(), slbM)
if err != nil {
return fmt.Errorf("create internet slb error: %s", err.Error())
if err := f.Client.CloudClient.FindLoadBalancerByName(slbM); err != nil {
return err
}
if slbM.LoadBalancerAttribute.LoadBalancerId == "" {
if err := f.Client.CloudClient.CreateLoadBalancer(context.TODO(), slbM); err != nil {
return fmt.Errorf("create internet slb error: %s", err.Error())
}
}
options.TestConfig.InternetLoadBalancerID = slbM.LoadBalancerAttribute.LoadBalancerId
f.CreatedResource[options.TestConfig.InternetLoadBalancerID] = SLBResource
Expand Down Expand Up @@ -146,29 +150,48 @@ func (f *Framework) CreateCloudResource() error {
LoadBalancerSpec: model.S1Small,
RegionId: region,
VSwitchId: vswId,
LoadBalancerName: client.Namespace + "-intranet-slb",
LoadBalancerName: fmt.Sprintf("%s-%s-slb", options.TestConfig.ClusterId, "intranet"),
},
}
if err := f.Client.CloudClient.CreateLoadBalancer(context.TODO(), slbM); err != nil {
return fmt.Errorf("create intranet slb error: %s", err.Error())
if err := f.Client.CloudClient.FindLoadBalancerByName(slbM); err != nil {
return err
}
if slbM.LoadBalancerAttribute.LoadBalancerId == "" {
if err := f.Client.CloudClient.CreateLoadBalancer(context.TODO(), slbM); err != nil {
return fmt.Errorf("create intranet slb error: %s", err.Error())
}
}
options.TestConfig.IntranetLoadBalancerID = slbM.LoadBalancerAttribute.LoadBalancerId
f.CreatedResource[options.TestConfig.IntranetLoadBalancerID] = SLBResource
}

if options.TestConfig.AclID == "" {
aclId, err := f.Client.CloudClient.CreateAccessControlList(context.TODO(), client.Namespace+"-acl-1")
aclName := fmt.Sprintf("%s-acl-%s", options.TestConfig.ClusterId, "a")
aclId, err := f.Client.CloudClient.DescribeAccessControlList(context.TODO(), aclName)
if err != nil {
return fmt.Errorf("CreateAccessControlList error: %s", err.Error())
return fmt.Errorf("DescribeAccessControlList error: %s", err.Error())
}
if aclId == "" {
aclId, err = f.Client.CloudClient.CreateAccessControlList(context.TODO(), aclName)
if err != nil {
return fmt.Errorf("CreateAccessControlList error: %s", err.Error())
}
}
options.TestConfig.AclID = aclId
f.CreatedResource[aclId] = ACLResource
}

if options.TestConfig.AclID2 == "" {
aclId, err := f.Client.CloudClient.CreateAccessControlList(context.TODO(), client.Namespace+"-acl-2")
aclName := fmt.Sprintf("%s-acl-%s", options.TestConfig.ClusterId, "b")
aclId, err := f.Client.CloudClient.DescribeAccessControlList(context.TODO(), aclName)
if err != nil {
return fmt.Errorf("CreateAccessControlList error: %s", err.Error())
return fmt.Errorf("DescribeAccessControlList error: %s", err.Error())
}
if aclId == "" {
aclId, err = f.Client.CloudClient.CreateAccessControlList(context.TODO(), aclName)
if err != nil {
return fmt.Errorf("CreateAccessControlList error: %s", err.Error())
}
}
options.TestConfig.AclID2 = aclId
f.CreatedResource[aclId] = ACLResource
Expand Down
32 changes: 16 additions & 16 deletions e2e/options/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,22 +52,22 @@ func (e *E2EConfig) BindFlags() {
flag.StringVar(&e.RegionId, "region-id", "", "the region id of cluster")
flag.StringVar(&e.ClusterId, "cluster-id", "", "the id of cluster which is used to run e2e test")
flag.BoolVar(&e.AllowCreateCloudResource, "allow-create-cloud-resources", false, "whether allow to create cloud resources, including the Kubernetes Cluster, SLB, ECS, etc.")
flag.StringVar(&TestConfig.EipLoadBalancerID, "eip-lb-id", "", "reused intranet slb id which has eip")
flag.StringVar(&TestConfig.VPCLoadBalancerID, "vpc-lb-id", "", "reused intranet slb id which in other vpc")
flag.StringVar(&TestConfig.MasterZoneID, "master-zone-id", "", "master zone id")
flag.StringVar(&TestConfig.SlaveZoneID, "slave-zone-id", "", "slave zone id")
flag.StringVar(&TestConfig.ResourceGroupID, "resource-group-id", "", "resource group id, do not use the resource group id of the ack cluster")
flag.StringVar(&TestConfig.Network, "network", "", "the network type of kubernetes, values: flannel or terway")
flag.StringVar(&TestConfig.IntranetLoadBalancerID, "intranet-lb-id", "", "reused intranet slb id")
flag.StringVar(&TestConfig.InternetLoadBalancerID, "internet-lb-id", "", "reused internet slb id")
flag.StringVar(&TestConfig.AclID, "acl-id", "", "acl id")
flag.StringVar(&TestConfig.AclID2, "acl-id-2", "", "acl id")
flag.StringVar(&TestConfig.VSwitchID, "vswitch-id", "", "vsw-id")
flag.StringVar(&TestConfig.VSwitchID2, "vswitch-id-2", "", "vsw-id")
flag.StringVar(&TestConfig.CertID, "cert-id", "", "cert id")
flag.StringVar(&TestConfig.CertID2, "cert-id-2", "", "cert id")
flag.StringVar(&TestConfig.VServerGroupID, "vserver-group-id", "", "vserver group id")
flag.StringVar(&TestConfig.VServerGroupID2, "vserver-group-id-2", "", "vserver group id")
flag.StringVar(&e.EipLoadBalancerID, "eip-lb-id", "", "reused intranet slb id which has eip")
flag.StringVar(&e.VPCLoadBalancerID, "vpc-lb-id", "", "reused intranet slb id which in other vpc")
flag.StringVar(&e.MasterZoneID, "master-zone-id", "", "master zone id")
flag.StringVar(&e.SlaveZoneID, "slave-zone-id", "", "slave zone id")
flag.StringVar(&e.ResourceGroupID, "resource-group-id", "", "resource group id, do not use the resource group id of the ack cluster")
flag.StringVar(&e.Network, "network", "", "the network type of kubernetes, values: flannel or terway")
flag.StringVar(&e.IntranetLoadBalancerID, "intranet-lb-id", "", "reused intranet slb id")
flag.StringVar(&e.InternetLoadBalancerID, "internet-lb-id", "", "reused internet slb id")
flag.StringVar(&e.AclID, "acl-id", "", "acl id")
flag.StringVar(&e.AclID2, "acl-id-2", "", "acl id")
flag.StringVar(&e.VSwitchID, "vswitch-id", "", "vsw-id")
flag.StringVar(&e.VSwitchID2, "vswitch-id-2", "", "vsw-id")
flag.StringVar(&e.CertID, "cert-id", "", "cert id")
flag.StringVar(&e.CertID2, "cert-id-2", "", "cert id")
flag.StringVar(&e.VServerGroupID, "vserver-group-id", "", "vserver group id")
flag.StringVar(&e.VServerGroupID2, "vserver-group-id-2", "", "vserver group id")
}

func (e *E2EConfig) Validate() error {
Expand Down
4 changes: 0 additions & 4 deletions e2e/testcase/node/node.go
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package node

import (
"time"

"github.com/onsi/ginkgo"
"github.com/onsi/gomega"
v1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -31,7 +29,6 @@ func RunNodeControllerTestCases(f *framework.Framework) {
_, err = f.Client.KubeClient.PatchNodeStatus(oldNode, newNode)
gomega.Expect(err).To(gomega.BeNil())

time.Sleep(5 * time.Minute)
err = f.ExpectNodeEqual()
gomega.Expect(err).To(gomega.BeNil())
})
Expand All @@ -42,7 +39,6 @@ func RunNodeControllerTestCases(f *framework.Framework) {
err = f.Client.KubeClient.LabelNode(oldNode.Name, v1.LabelInstanceType, "test-type")
gomega.Expect(err).To(gomega.BeNil())

time.Sleep(5 * time.Minute)
err = f.ExpectNodeEqual()
gomega.Expect(err).To(gomega.BeNil())
})
Expand Down
9 changes: 7 additions & 2 deletions e2e/testcase/route/route.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,12 @@ import (
"github.com/onsi/ginkgo"
"github.com/onsi/gomega"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/cloud-provider-alibaba-cloud/e2e/framework"
"k8s.io/cloud-provider-alibaba-cloud/e2e/options"
ctrlCfg "k8s.io/cloud-provider-alibaba-cloud/pkg/config"
"strings"
"time"
)

func RunRouteControllerTestCases(f *framework.Framework) {
Expand Down Expand Up @@ -46,8 +48,11 @@ func RunRouteControllerTestCases(f *framework.Framework) {
routes, err := f.Client.CloudClient.ListRoute(context.TODO(), resp.RouteTableId)
gomega.Expect(err).To(gomega.BeNil())
for _, t := range routes {
err = f.Client.CloudClient.DeleteRoute(context.TODO(), resp.RouteTableId,
t.ProviderId, t.DestinationCIDR)
err = wait.PollImmediate(5*time.Second, 2*time.Minute, func() (done bool, err error) {
retErr := f.Client.CloudClient.DeleteRoute(context.TODO(), resp.RouteTableId,
t.ProviderId, t.DestinationCIDR)
return retErr == nil, nil
})
gomega.Expect(err).To(gomega.BeNil())
}
_, err = f.Client.CloudClient.DeleteRouteTable(context.TODO(), resp.RouteTableId)
Expand Down
Loading

0 comments on commit 21173d2

Please sign in to comment.