diff --git a/.golangci.yml b/.golangci.yml index eec0fea19..c8d26644d 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -32,4 +32,10 @@ linters: - gosec - goimports - vet - - revive \ No newline at end of file + - revive + +issues: + exclude-rules: + - linters: + - staticcheck + text: "SA1019: lbs.RelaxScaleValidation" \ No newline at end of file diff --git a/pkg/controllers/networkinfo/networkinfo_controller.go b/pkg/controllers/networkinfo/networkinfo_controller.go index 06e62de74..5669a2563 100644 --- a/pkg/controllers/networkinfo/networkinfo_controller.go +++ b/pkg/controllers/networkinfo/networkinfo_controller.go @@ -78,7 +78,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request) log.Error(err, "failed to check if namespace is shared", "Namespace", obj.GetNamespace()) return common.ResultRequeue, err } - if !isShared { + if r.Service.NSXConfig.NsxConfig.UseAVILoadBalancer && !isShared { err = r.Service.CreateOrUpdateAVIRule(createdVpc, obj.Namespace) if err != nil { state := &v1alpha1.VPCState{ @@ -116,7 +116,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request) // if lb vpc enabled, read avi subnet path and cidr // nsx bug, if set LoadBalancerVpcEndpoint.Enabled to false, when read this vpc back, // LoadBalancerVpcEndpoint.Enabled will become a nil pointer. - if createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled { + if r.Service.NSXConfig.NsxConfig.UseAVILoadBalancer && createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled { path, cidr, err = r.Service.GetAVISubnetInfo(*createdVpc) if err != nil { log.Error(err, "failed to read lb subnet path and cidr", "VPC", createdVpc.Id) @@ -139,7 +139,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request) LoadBalancerIPAddresses: cidr, PrivateIPv4CIDRs: nc.PrivateIPv4CIDRs, } - updateSuccess(r, &ctx, obj, r.Client, state, nc.Name, path) + updateSuccess(r, &ctx, obj, r.Client, state, nc.Name, path, r.Service.GetNSXLBSPath(*createdVpc.Id)) } else { if controllerutil.ContainsFinalizer(obj, commonservice.NetworkInfoFinalizerName) { metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeNetworkInfo) diff --git a/pkg/controllers/networkinfo/networkinfo_utils.go b/pkg/controllers/networkinfo/networkinfo_utils.go index 076301185..e34164509 100644 --- a/pkg/controllers/networkinfo/networkinfo_utils.go +++ b/pkg/controllers/networkinfo/networkinfo_utils.go @@ -29,10 +29,10 @@ func updateFail(r *NetworkInfoReconciler, c *context.Context, o *v1alpha1.Networ } func updateSuccess(r *NetworkInfoReconciler, c *context.Context, o *v1alpha1.NetworkInfo, client client.Client, - vpcState *v1alpha1.VPCState, ncName string, subnetPath string) { + vpcState *v1alpha1.VPCState, ncName string, subnetPath string, nsxLBSPath string) { setNetworkInfoVPCStatus(c, o, client, vpcState) // ako needs to know the avi subnet path created by nsx - setVPCNetworkConfigurationStatus(c, client, ncName, vpcState.Name, subnetPath) + setVPCNetworkConfigurationStatus(c, client, ncName, vpcState.Name, subnetPath, nsxLBSPath) r.Recorder.Event(o, v1.EventTypeNormal, common.ReasonSuccessfulUpdate, "NetworkInfo CR has been successfully updated") metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateSuccessTotal, common.MetricResTypeNetworkInfo) } @@ -59,7 +59,7 @@ func setNetworkInfoVPCStatus(ctx *context.Context, networkInfo *v1alpha1.Network } } -func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string) { +func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string, nsxLBSPath string) { // read v1alpha1.VPCNetworkConfiguration by ncName nc := &v1alpha1.VPCNetworkConfiguration{} err := client.Get(*ctx, apitypes.NamespacedName{Name: ncName}, nc) @@ -67,8 +67,9 @@ func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client log.Error(err, "failed to get VPCNetworkConfiguration", "Name", ncName) } createdVPCInfo := &v1alpha1.VPCInfo{ - Name: vpcName, - AVISESubnetPath: aviSubnetPath, + Name: vpcName, + AVISESubnetPath: aviSubnetPath, + NSXLoadBalancerPath: nsxLBSPath, } // iterate through VPCNetworkConfiguration.Status.VPCs, if vpcName already exists, update it for i, vpc := range nc.Status.VPCs { diff --git a/pkg/nsx/client.go b/pkg/nsx/client.go index 9bc961f36..ea1e845ed 100644 --- a/pkg/nsx/client.go +++ b/pkg/nsx/client.go @@ -85,6 +85,7 @@ type Client struct { SubnetsClient vpcs.SubnetsClient RealizedStateClient realized_state.RealizedEntitiesClient IPAddressAllocationClient vpcs.IpAddressAllocationsClient + VPCLBSClient vpcs.VpcLbsClient NSXChecker NSXHealthChecker NSXVerChecker NSXVersionChecker @@ -165,6 +166,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { subnetStatusClient := subnets.NewStatusClient(restConnector(cluster)) realizedStateClient := realized_state.NewRealizedEntitiesClient(restConnector(cluster)) ipAddressAllocationClient := vpcs.NewIpAddressAllocationsClient(restConnector(cluster)) + vpcLBSClient := vpcs.NewVpcLbsClient(restConnector(cluster)) vpcSecurityClient := vpcs.NewSecurityPoliciesClient(restConnector(cluster)) vpcRuleClient := vpc_sp.NewRulesClient(restConnector(cluster)) @@ -206,6 +208,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { SubnetStatusClient: subnetStatusClient, VPCSecurityClient: vpcSecurityClient, VPCRuleClient: vpcRuleClient, + VPCLBSClient: vpcLBSClient, NSXChecker: *nsxChecker, NSXVerChecker: *nsxVersionChecker, diff --git a/pkg/nsx/services/common/types.go b/pkg/nsx/services/common/types.go index c12d3d04c..81bae7e20 100644 --- a/pkg/nsx/services/common/types.go +++ b/pkg/nsx/services/common/types.go @@ -28,6 +28,7 @@ const ( TagScopeNCPVIFProjectUID string = "ncp/vif_project_uid" TagScopeNCPPod string = "ncp/pod" TagScopeNCPVNETInterface string = "ncp/vnet_interface" + TagScopeCreatedFor string = "nsx-op/created_for" TagScopeVersion string = "nsx-op/version" TagScopeCluster string = "nsx-op/cluster" TagScopeNamespace string = "nsx-op/namespace" @@ -74,6 +75,7 @@ const ( TagValueGroupSource string = "source" TagValueGroupDestination string = "destination" TagValueGroupAvi string = "avi" + TagValueSLB string = "SLB" AnnotationVPCNetworkConfig string = "nsx.vmware.com/vpc_network_config" AnnotationSharedVPCNamespace string = "nsx.vmware.com/shared_vpc_namespace" AnnotationDefaultNetworkConfig string = "nsx.vmware.com/default" @@ -147,6 +149,7 @@ var ( ResourceTypeVpc = "Vpc" ResourceTypeSubnetPort = "VpcSubnetPort" ResourceTypeVirtualMachine = "VirtualMachine" + ResourceTypeLBService = "LBService" ResourceTypeShare = "Share" ResourceTypeSharedResource = "SharedResource" ResourceTypeChildSharedResource = "ChildSharedResource" diff --git a/pkg/nsx/services/common/wrap.go b/pkg/nsx/services/common/wrap.go new file mode 100644 index 000000000..a2c96d3a0 --- /dev/null +++ b/pkg/nsx/services/common/wrap.go @@ -0,0 +1,84 @@ +package common + +import ( + "github.com/openlyinc/pointy" + "github.com/vmware/vsphere-automation-sdk-go/runtime/data" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" +) + +// WrapInfra TODO(gran) refactor existing code in other package +func (service *Service) WrapInfra(children []*data.StructValue) (*model.Infra, error) { + // This is the outermost layer of the hierarchy infra client. + // It doesn't need ID field. + resourceType := ResourceTypeInfra + infraObj := model.Infra{ + Children: children, + ResourceType: &resourceType, + } + return &infraObj, nil +} + +func (service *Service) WrapOrgRoot(children []*data.StructValue) (*model.OrgRoot, error) { + resourceType := ResourceTypeOrgRoot + orgRootObj := model.OrgRoot{ + Children: children, + ResourceType: &resourceType, + } + return &orgRootObj, nil +} + +func (service *Service) WrapOrg(org string, children []*data.StructValue) ([]*data.StructValue, error) { + targetType := ResourceTypeOrg + return wrapChildResourceReference(targetType, org, children) +} + +func (service *Service) WrapProject(nsxtProject string, children []*data.StructValue) ([]*data.StructValue, error) { + targetType := ResourceTypeProject + return wrapChildResourceReference(targetType, nsxtProject, children) +} + +func wrapChildResourceReference(targetType, id string, children []*data.StructValue) ([]*data.StructValue, error) { + resourceType := ResourceTypeChildResourceReference + childProject := model.ChildResourceReference{ + Id: &id, + ResourceType: resourceType, + TargetType: &targetType, + Children: children, + } + dataValue, errors := NewConverter().ConvertToVapi(childProject, childProject.GetType__()) + if len(errors) > 0 { + return nil, errors[0] + } + return []*data.StructValue{dataValue.(*data.StructValue)}, nil + +} + +func (service *Service) WrapVPC(vpc *model.Vpc) ([]*data.StructValue, error) { + vpc.ResourceType = pointy.String(ResourceTypeVpc) + childVpc := model.ChildVpc{ + Id: vpc.Id, + MarkedForDelete: vpc.MarkedForDelete, + ResourceType: "ChildVpc", + Vpc: vpc, + } + dataValue, errs := NewConverter().ConvertToVapi(childVpc, childVpc.GetType__()) + if len(errs) > 0 { + return nil, errs[0] + } + return []*data.StructValue{dataValue.(*data.StructValue)}, nil +} + +func (service *Service) WrapLBS(lbs *model.LBService) ([]*data.StructValue, error) { + lbs.ResourceType = pointy.String(ResourceTypeLBService) + childLBService := model.ChildLBService{ + Id: lbs.Id, + MarkedForDelete: lbs.MarkedForDelete, + ResourceType: "ChildLBService", + LbService: lbs, + } + dataValue, errs := NewConverter().ConvertToVapi(childLBService, childLBService.GetType__()) + if len(errs) > 0 { + return nil, errs[0] + } + return []*data.StructValue{dataValue.(*data.StructValue)}, nil +} diff --git a/pkg/nsx/services/realizestate/realize_state.go b/pkg/nsx/services/realizestate/realize_state.go index 8fef3bf8a..3fbc1ea35 100644 --- a/pkg/nsx/services/realizestate/realize_state.go +++ b/pkg/nsx/services/realizestate/realize_state.go @@ -49,7 +49,7 @@ func (service *RealizeStateService) CheckRealizeState(backoff wait.Backoff, inte return err } for _, result := range results.Results { - if *result.EntityType != entityType { + if entityType != "" && result.EntityType != nil && *result.EntityType != entityType { continue } if *result.State == model.GenericPolicyRealizedResource_STATE_REALIZED { diff --git a/pkg/nsx/services/vpc/builder.go b/pkg/nsx/services/vpc/builder.go index 5189ac73e..04d3c4227 100644 --- a/pkg/nsx/services/vpc/builder.go +++ b/pkg/nsx/services/vpc/builder.go @@ -51,7 +51,7 @@ func buildPrivateIpBlock(networkInfo *v1alpha1.NetworkInfo, nsObj *v1.Namespace, } func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNetworkConfigInfo, cluster string, pathMap map[string]string, - nsxVPC *model.Vpc) (*model.Vpc, + nsxVPC *model.Vpc, useAVILB bool) (*model.Vpc, error) { vpc := &model.Vpc{} if nsxVPC != nil { @@ -61,7 +61,7 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe return nil, nil } // for updating vpc case, use current vpc id, name - vpc = nsxVPC + *vpc = *nsxVPC } else { // for creating vpc case, fill in vpc properties based on networkconfig vpcName := util.GenerateDisplayName("", "vpc", obj.GetNamespace(), "", cluster) @@ -76,7 +76,10 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe }, } vpc.SiteInfos = siteInfos - vpc.LoadBalancerVpcEndpoint = &model.LoadBalancerVPCEndpoint{Enabled: &DefaultLoadBalancerVPCEndpointEnabled} + if useAVILB { + loadBalancerVPCEndpointEnabled := true + vpc.LoadBalancerVpcEndpoint = &model.LoadBalancerVPCEndpoint{Enabled: &loadBalancerVPCEndpointEnabled} + } vpc.Tags = util.BuildBasicTags(cluster, obj, nsObj.UID) } @@ -89,3 +92,21 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe return vpc, nil } + +func buildNSXLBS(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, cluster, lbsSize, vpcPath string, relaxScaleValidation *bool) (*model.LBService, error) { + lbs := &model.LBService{} + lbsName := util.GenerateDisplayName("", "vpc", nsObj.GetName(), "", cluster) + // Use VPC id for auto-created LBS id + lbs.Id = common.String(string(nsObj.GetUID())) + lbs.DisplayName = &lbsName + lbs.Tags = util.BuildBasicTags(cluster, obj, nsObj.GetUID()) + // "created_for" is required by NCP, and "lb_t1_link_ip" is not needed for VPC + lbs.Tags = append(lbs.Tags, model.Tag{ + Scope: common.String(common.TagScopeCreatedFor), + Tag: common.String(common.TagValueSLB), + }) + lbs.Size = &lbsSize + lbs.ConnectivityPath = &vpcPath + lbs.RelaxScaleValidation = relaxScaleValidation + return lbs, nil +} diff --git a/pkg/nsx/services/vpc/builder_test.go b/pkg/nsx/services/vpc/builder_test.go new file mode 100644 index 000000000..06d93f34b --- /dev/null +++ b/pkg/nsx/services/vpc/builder_test.go @@ -0,0 +1,79 @@ +package vpc + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" +) + +func Test_buildNSXLBS(t *testing.T) { + type args struct { + obj *v1alpha1.NetworkInfo + nsObj *v1.Namespace + cluster string + lbsSize string + vpcPath string + relaxScaleValidation *bool + } + tests := []struct { + name string + args args + want *model.LBService + wantErr assert.ErrorAssertionFunc + }{ + { + name: "1", + args: args{ + obj: &v1alpha1.NetworkInfo{ + ObjectMeta: metav1.ObjectMeta{Namespace: "ns1"}, + VPCs: nil, + }, + nsObj: &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "ns1", UID: "nsuid1"}, + }, + cluster: "cluster1", + lbsSize: model.LBService_SIZE_SMALL, + vpcPath: "/vpc1", + relaxScaleValidation: nil, + }, + want: &model.LBService{ + Id: common.String("nsuid1"), + DisplayName: common.String("vpc-cluster1--ns1"), + Tags: []model.Tag{ + { + Scope: common.String(common.TagScopeCluster), + Tag: common.String("cluster1"), + }, + { + Scope: common.String(common.TagScopeVersion), + Tag: common.String(strings.Join(common.TagValueVersion, ".")), + }, + {Scope: common.String(common.TagScopeNamespace), Tag: common.String("ns1")}, + {Scope: common.String(common.TagScopeNamespaceUID), Tag: common.String("nsuid1")}, + {Scope: common.String(common.TagScopeCreatedFor), Tag: common.String(common.TagValueSLB)}, + }, + Size: common.String(model.LBService_SIZE_SMALL), + ConnectivityPath: common.String("/vpc1"), + RelaxScaleValidation: nil, + }, + wantErr: assert.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := buildNSXLBS(tt.args.obj, tt.args.nsObj, tt.args.cluster, tt.args.lbsSize, tt.args.vpcPath, tt.args.relaxScaleValidation) + if !tt.wantErr(t, err, fmt.Sprintf("buildNSXLBS(%v, %v, %v, %v, %v, %v)", tt.args.obj, tt.args.nsObj, tt.args.cluster, tt.args.lbsSize, tt.args.vpcPath, tt.args.relaxScaleValidation)) { + return + } + assert.Equalf(t, tt.want, got, "buildNSXLBS(%v, %v, %v, %v, %v, %v)", tt.args.obj, tt.args.nsObj, tt.args.cluster, tt.args.lbsSize, tt.args.vpcPath, tt.args.relaxScaleValidation) + }) + } +} diff --git a/pkg/nsx/services/vpc/store.go b/pkg/nsx/services/vpc/store.go index 6c5d90610..4b3ca419d 100644 --- a/pkg/nsx/services/vpc/store.go +++ b/pkg/nsx/services/vpc/store.go @@ -13,6 +13,8 @@ func keyFunc(obj interface{}) (string, error) { switch v := obj.(type) { case *model.Vpc: return *v.Id, nil + case *model.LBService: + return *v.Id, nil case *model.IpAddressBlock: return generateIPBlockKey(*v), nil default: @@ -27,6 +29,8 @@ func indexFunc(obj interface{}) ([]string, error) { switch o := obj.(type) { case *model.Vpc: return filterTag(o.Tags), nil + case *model.LBService: + return filterTag(o.Tags), nil case *model.IpAddressBlock: return filterTag(o.Tags), nil default: @@ -148,6 +152,41 @@ func (is *IPBlockStore) GetByIndex(index string, value string) *model.IpAddressB return block } +// LBSStore is a store for LBS +type LBSStore struct { + common.ResourceStore +} + +func (ls *LBSStore) Apply(i interface{}) error { + if i == nil { + return nil + } + lbs := i.(*model.LBService) + if lbs.MarkedForDelete != nil && *lbs.MarkedForDelete { + err := ls.Delete(lbs) + log.V(1).Info("delete LBS from store", "LBS", lbs) + if err != nil { + return err + } + } else { + err := ls.Add(lbs) + log.V(1).Info("add LBS to store", "LBS", lbs) + if err != nil { + return err + } + } + return nil +} + +func (ls *LBSStore) GetByKey(key string) *model.LBService { + obj := ls.ResourceStore.GetByKey(key) + if obj != nil { + lbs := obj.(*model.LBService) + return lbs + } + return nil +} + // keyFuncAVI is used to get the key of a AVI rule related resource func keyFuncAVI(obj interface{}) (string, error) { switch v := obj.(type) { diff --git a/pkg/nsx/services/vpc/store_test.go b/pkg/nsx/services/vpc/store_test.go index 2e6601615..8e0ab0734 100644 --- a/pkg/nsx/services/vpc/store_test.go +++ b/pkg/nsx/services/vpc/store_test.go @@ -8,6 +8,7 @@ import ( "github.com/agiledragon/gomonkey/v2" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/vmware/vsphere-automation-sdk-go/runtime/bindings" "github.com/vmware/vsphere-automation-sdk-go/runtime/data" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" @@ -32,20 +33,6 @@ func (qIface *fakeQueryClient) List(_ string, _ *string, _ *string, _ *int64, _ }, nil } -func Test_IndexFunc(t *testing.T) { - mId, mTag, mScope := "test_id", "test_tag", "nsx-op/namespace_uid" - v := &model.Vpc{ - Id: &mId, - Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, - } - t.Run("1", func(t *testing.T) { - got, _ := indexFunc(v) - if !reflect.DeepEqual(got, []string{"test_tag"}) { - t.Errorf("NSCRUIDScopeIndexFunc() = %v, want %v", got, model.Tag{Tag: &mTag, Scope: &mScope}) - } - }) -} - func Test_filterTag(t *testing.T) { mTag, mScope := "test_tag", "nsx-op/namespace_uid" mTag2, mScope2 := "test_tag", "nsx" @@ -74,17 +61,6 @@ func Test_filterTag(t *testing.T) { } } -func Test_KeyFunc(t *testing.T) { - Id := "test_id" - v := &model.Vpc{Id: &Id} - t.Run("1", func(t *testing.T) { - got, _ := keyFunc(v) - if got != "test_id" { - t.Errorf("keyFunc() = %v, want %v", got, "test_id") - } - }) -} - func Test_InitializeVPCStore(t *testing.T) { config2 := nsx.NewConfig("localhost", "1", "1", []string{}, 10, 3, 20, 20, true, true, true, ratelimiter.AIMD, nil, nil, []string{}) cluster, _ := nsx.NewCluster(config2) @@ -337,3 +313,129 @@ func TestSecurityPolicyStore_GetByKey(t *testing.T) { sp = spStore.GetByKey(path2) assert.Equal(t, sp.Path, sp2.Path) } + +func Test_keyFunc(t *testing.T) { + id := "test_id" + type args struct { + obj interface{} + } + tests := []struct { + name string + args args + want string + wantErr assert.ErrorAssertionFunc + }{ + { + name: "vpc", + args: args{obj: &model.Vpc{Id: &id}}, + want: id, + wantErr: assert.NoError, + }, + { + name: "lbs", + args: args{obj: &model.LBService{Id: &id}}, + want: id, + wantErr: assert.NoError, + }, + { + name: "invalid", + args: args{obj: &model.AntreaTraceflowConfig{Id: &id}}, + want: "", + wantErr: assert.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := keyFunc(tt.args.obj) + if !tt.wantErr(t, err, fmt.Sprintf("keyFunc(%v)", tt.args.obj)) { + return + } + assert.Equalf(t, tt.want, got, "keyFunc(%v)", tt.args.obj) + }) + } +} + +func Test_indexFunc(t *testing.T) { + mId, mTag, mScope := "test_id", "test_tag", "nsx-op/namespace_uid" + type args struct { + obj interface{} + } + tests := []struct { + name string + args args + want []string + wantErr assert.ErrorAssertionFunc + }{ + { + name: "vpc", + args: args{obj: &model.Vpc{ + Id: &mId, + Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, + }}, + want: []string{"test_tag"}, + wantErr: assert.NoError, + }, + { + name: "lbs", + args: args{obj: &model.LBService{ + Id: &mId, + Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, + }}, + want: []string{"test_tag"}, + wantErr: assert.NoError, + }, + { + name: "lbsnotag", + args: args{obj: &model.LBService{ + Id: &mId, + Tags: []model.Tag{}, + }}, + want: []string{}, + wantErr: assert.NoError, + }, + { + name: "invalid", + args: args{obj: &model.AntreaTraceflowConfig{ + Id: &mId, + Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, + }}, + want: []string{}, + wantErr: assert.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := indexFunc(tt.args.obj) + if !tt.wantErr(t, err, fmt.Sprintf("indexFunc(%v)", tt.args.obj)) { + return + } + assert.Equalf(t, tt.want, got, "indexFunc(%v)", tt.args.obj) + }) + } +} + +func TestLBSStore_CRUD(t *testing.T) { + vpcCacheIndexer := cache.NewIndexer(keyFunc, cache.Indexers{}) + resourceStore := common.ResourceStore{ + Indexer: vpcCacheIndexer, + BindingType: model.LBServiceBindingType(), + } + ls := &LBSStore{ + ResourceStore: resourceStore, + } + lbs1 := &model.LBService{Id: common.String("1")} + lbs2 := &model.LBService{Id: common.String("2")} + require.NoError(t, ls.Apply(lbs1)) + require.Equal(t, 1, len(ls.List())) + require.True(t, reflect.DeepEqual(lbs1, ls.GetByKey("1"))) + require.NoError(t, ls.Apply(lbs2)) + require.Equal(t, 2, len(ls.List())) + lbs2.MarkedForDelete = common.Bool(true) + require.NoError(t, ls.Apply(lbs2)) + require.Equal(t, 1, len(ls.List())) + require.Nil(t, ls.GetByKey("2")) + defer func() { + require.NotNil(t, recover()) + }() + ls.Apply(&model.AntreaTraceflowConfig{Id: common.String("invalid")}) +} diff --git a/pkg/nsx/services/vpc/vpc.go b/pkg/nsx/services/vpc/vpc.go index 415b92773..577f7a612 100644 --- a/pkg/nsx/services/vpc/vpc.go +++ b/pkg/nsx/services/vpc/vpc.go @@ -15,7 +15,6 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/cache" - "k8s.io/client-go/util/retry" "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" "github.com/vmware-tanzu/nsx-operator/pkg/logger" @@ -30,6 +29,7 @@ const ( AviSEIngressAllowRuleId = "avi-se-ingress-allow-rule" VPCAviSEGroupId = "avi-se-vms" VpcDefaultSecurityPolicyId = "default-layer3-section" + VPCKey = "/orgs/%s/projects/%s/vpcs/%s" GroupKey = "/orgs/%s/projects/%s/vpcs/%s/groups/%s" SecurityPolicyKey = "/orgs/%s/projects/%s/vpcs/%s/security-policies/%s" RuleKey = "/orgs/%s/projects/%s/vpcs/%s/security-policies/%s/rules/%s" @@ -41,8 +41,9 @@ var ( ResourceTypeVPC = common.ResourceTypeVpc NewConverter = common.NewConverter - MarkedForDelete = true - enableAviAllowRule = false + MarkedForDelete = true + enableAviAllowRule = false + EnforceRevisionCheckParam = false ) type VPCNetworkInfoStore struct { @@ -58,6 +59,7 @@ type VPCNsNetworkConfigStore struct { type VPCService struct { common.Service VpcStore *VPCStore + LbsStore *LBSStore IpblockStore *IPBlockStore VPCNetworkConfigStore VPCNetworkInfoStore VPCNSNetworkConfigStore VPCNsNetworkConfigStore @@ -161,6 +163,10 @@ func InitializeVPC(service common.Service) (*VPCService, error) { Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), BindingType: model.VpcBindingType(), }} + VPCService.LbsStore = &LBSStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), + BindingType: model.LBServiceBindingType(), + }} VPCService.IpblockStore = &IPBlockStore{ResourceStore: common.ResourceStore{ Indexer: cache.NewIndexer(keyFunc, cache.Indexers{ @@ -173,8 +179,10 @@ func InitializeVPC(service common.Service) (*VPCService, error) { VPCService.VPCNSNetworkConfigStore = VPCNsNetworkConfigStore{ VPCNSNetworkConfigMap: make(map[string]string), } - //initialize vpc store and ip blocks store + //initialize vpc store, lbs store and ip blocks store go VPCService.InitializeResourceStore(&wg, fatalErrors, common.ResourceTypeVpc, nil, VPCService.VpcStore) + wg.Add(1) + go VPCService.InitializeResourceStore(&wg, fatalErrors, common.ResourceTypeLBService, nil, VPCService.LbsStore) go VPCService.InitializeResourceStore(&wg, fatalErrors, common.ResourceTypeIPBlock, nil, VPCService.IpblockStore) //initalize avi rule related store @@ -251,6 +259,10 @@ func (s *VPCService) DeleteVPC(path string) error { err = nsxutil.NSXApiError(err) return err } + lbs := s.LbsStore.GetByKey(pathInfo.VPCID) + if lbs != nil { + s.LbsStore.Delete(lbs) + } vpc.MarkedForDelete = &MarkedForDelete if err := s.VpcStore.Apply(vpc); err != nil { return err @@ -556,7 +568,7 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo) (*model.Vpc, * nsxVPC = nil } - createdVpc, err := buildNSXVPC(obj, nsObj, nc, s.NSXConfig.Cluster, paths, nsxVPC) + createdVpc, err := buildNSXVPC(obj, nsObj, nc, s.NSXConfig.Cluster, paths, nsxVPC, s.NSXConfig.NsxConfig.UseAVILoadBalancer) if err != nil { log.Error(err, "failed to build NSX VPC object") return nil, nil, err @@ -568,8 +580,26 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo) (*model.Vpc, * return existingVPC[0], &nc, nil } + // build NSX LBS + var createdLBS *model.LBService + if s.NSXConfig.NsxConfig.NSXLBEnabled() { + lbsSize := s.NSXConfig.NsxConfig.GetNSXLBSize() + vpcPath := fmt.Sprintf(VPCKey, nc.Org, nc.NsxtProject, nc.Name) + var relaxScaleValidation *bool + if s.NSXConfig.NsxConfig.RelaxNSXLBScaleValication { + relaxScaleValidation = common.Bool(true) + } + createdLBS, _ = buildNSXLBS(obj, nsObj, s.NSXConfig.Cluster, lbsSize, vpcPath, relaxScaleValidation) + } + // build HAPI request + orgRoot, err := s.WrapHierarchyVPC(nc.Org, nc.NsxtProject, createdVpc, createdLBS) + if err != nil { + log.Error(err, "failed to build HAPI request") + return nil, nil, err + } + log.Info("creating NSX VPC", "VPC", *createdVpc.Id) - err = s.NSXClient.VPCClient.Patch(nc.Org, nc.NsxtProject, *createdVpc.Id, *createdVpc) + err = s.NSXClient.OrgRootClient.Patch(*orgRoot, &EnforceRevisionCheckParam) err = nsxutil.NSXApiError(err) if err != nil { log.Error(err, "failed to create VPC", "Project", nc.NsxtProject, "Namespace", obj.Namespace) @@ -596,12 +626,14 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo) (*model.Vpc, * return nil, nil, err } + log.V(2).Info("check VPC realization state", "VPC", *createdVpc.Id) realizeService := realizestate.InitializeRealizeState(s.Service) - if err = realizeService.CheckRealizeState(retry.DefaultRetry, *newVpc.Path, "RealizedLogicalRouter"); err != nil { + if err = realizeService.CheckRealizeState(util.NSXTDefaultRetry, *newVpc.Path, "RealizedLogicalRouter"); err != nil { log.Error(err, "failed to check VPC realization state", "VPC", *createdVpc.Id) if realizestate.IsRealizeStateError(err) { log.Error(err, "the created VPC is in error realization state, cleaning the resource", "VPC", *createdVpc.Id) - // delete the nsx vpc object and re-created in next loop + // delete the nsx vpc object and re-create it in the next loop + // TODO(gran) DeleteVPC will check VpcStore but new Vpc is not in store at this moment. Is it correct? if err := s.DeleteVPC(*newVpc.Path); err != nil { log.Error(err, "cleanup VPC failed", "VPC", *createdVpc.Id) return nil, nil, err @@ -611,6 +643,32 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo) (*model.Vpc, * } s.VpcStore.Add(&newVpc) + + // Check LBS realization + if createdLBS != nil { + newLBS, err := s.NSXClient.VPCLBSClient.Get(nc.Org, nc.NsxtProject, *createdVpc.Id, *createdLBS.Id) + if err != nil { + log.Error(err, "failed to read LBS object after creating or updating", "LBS", createdLBS.Id) + return nil, nil, err + } + s.LbsStore.Add(&newLBS) + + log.V(2).Info("check LBS realization state", "LBS", *createdLBS.Id) + realizeService := realizestate.InitializeRealizeState(s.Service) + if err = realizeService.CheckRealizeState(util.NSXTLBVSDefaultRetry, *newLBS.Path, ""); err != nil { + log.Error(err, "failed to check LBS realization state", "LBS", *createdLBS.Id) + if realizestate.IsRealizeStateError(err) { + log.Error(err, "the created LBS is in error realization state, cleaning the resource", "LBS", *createdLBS.Id) + // delete the nsx vpc object and re-create it in the next loop + if err := s.DeleteVPC(*newVpc.Path); err != nil { + log.Error(err, "cleanup VPC failed", "VPC", *createdVpc.Id) + return nil, nil, err + } + } + return nil, nil, err + } + } + return &newVpc, &nc, nil } @@ -902,3 +960,11 @@ func (service *VPCService) ListVPCInfo(ns string) []common.VPCResourceInfo { } return VPCInfoList } + +func (s *VPCService) GetNSXLBSPath(lbsId string) string { + vpcLBS := s.LbsStore.GetByKey(lbsId) + if vpcLBS == nil { + return "" + } + return *vpcLBS.Path +} diff --git a/pkg/nsx/services/vpc/wrap.go b/pkg/nsx/services/vpc/wrap.go new file mode 100644 index 000000000..8b16e771b --- /dev/null +++ b/pkg/nsx/services/vpc/wrap.go @@ -0,0 +1,41 @@ +package vpc + +import ( + "github.com/vmware/vsphere-automation-sdk-go/runtime/data" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" +) + +func (s *VPCService) WrapHierarchyVPC(org, nsxtProject string, vpc *model.Vpc, lbs *model.LBService) (*model.OrgRoot, error) { + if lbs != nil { + var vpcChildren []*data.StructValue + childrenLBS, err := s.WrapLBS(lbs) + if err != nil { + return nil, err + } + vpcChildren = append(vpcChildren, childrenLBS...) + vpc.Children = vpcChildren + } + var projectChildren []*data.StructValue + childrenVPC, err := s.WrapVPC(vpc) + if err != nil { + return nil, err + } + projectChildren = append(projectChildren, childrenVPC...) + + var orgChildren []*data.StructValue + childrenProject, err := s.WrapProject(nsxtProject, projectChildren) + if err != nil { + return nil, err + } + orgChildren = append(orgChildren, childrenProject...) + + var orgRootChildren []*data.StructValue + childrenOrg, err := s.WrapOrg(org, orgChildren) + if err != nil { + return nil, err + } + orgRootChildren = append(orgRootChildren, childrenOrg...) + + orgRoot, _ := s.WrapOrgRoot(orgRootChildren) + return orgRoot, nil +} diff --git a/pkg/nsx/services/vpc/wrap_test.go b/pkg/nsx/services/vpc/wrap_test.go new file mode 100644 index 000000000..b51338705 --- /dev/null +++ b/pkg/nsx/services/vpc/wrap_test.go @@ -0,0 +1,51 @@ +package vpc + +import ( + "fmt" + "testing" + + "github.com/openlyinc/pointy" + "github.com/stretchr/testify/assert" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" +) + +func TestVPCService_WrapHierarchyVPC(t *testing.T) { + type args struct { + org string + nsxtProject string + vpc *model.Vpc + lbs *model.LBService + } + tests := []struct { + name string + args args + want *model.OrgRoot + wantChildren int + wantErr assert.ErrorAssertionFunc + }{ + { + name: "test", + args: args{ + org: "testorg", + nsxtProject: "testproject", + vpc: &model.Vpc{}, + lbs: &model.LBService{}, + }, + want: &model.OrgRoot{ResourceType: pointy.String("OrgRoot")}, + wantChildren: 1, + wantErr: assert.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &VPCService{} + got, err := s.WrapHierarchyVPC(tt.args.org, tt.args.nsxtProject, tt.args.vpc, tt.args.lbs) + if !tt.wantErr(t, err, fmt.Sprintf("WrapHierarchyVPC(%v, %v, %v, %v)", tt.args.org, tt.args.nsxtProject, tt.args.vpc, tt.args.lbs)) { + return + } + assert.Equalf(t, tt.wantChildren, len(got.Children), "WrapHierarchyVPC children count") + got.Children = nil + assert.Equalf(t, tt.want, got, "WrapHierarchyVPC(%v, %v, %v, %v)", tt.args.org, tt.args.nsxtProject, tt.args.vpc, tt.args.lbs) + }) + } +} diff --git a/pkg/nsx/util/utils.go b/pkg/nsx/util/utils.go index d9d58745d..6fee23f9c 100644 --- a/pkg/nsx/util/utils.go +++ b/pkg/nsx/util/utils.go @@ -564,6 +564,8 @@ func CasttoPointer(obj interface{}) interface{} { return &v case model.Vpc: return &v + case model.LBService: + return &v case model.IpAddressPoolBlockSubnet: return &v case model.Group: diff --git a/pkg/util/retry.go b/pkg/util/retry.go new file mode 100644 index 000000000..131529a0e --- /dev/null +++ b/pkg/util/retry.go @@ -0,0 +1,21 @@ +package util + +import ( + "time" + + "k8s.io/apimachinery/pkg/util/wait" +) + +var NSXTDefaultRetry = wait.Backoff{ + Steps: 10, + Duration: 500 * time.Millisecond, + Factor: 1.0, + Jitter: 0.1, +} + +var NSXTLBVSDefaultRetry = wait.Backoff{ + Steps: 60, + Duration: 500 * time.Millisecond, + Factor: 1.0, + Jitter: 0.1, +}