Skip to content

Commit

Permalink
[VPC] Support NSXLB for VPC (vmware-tanzu#618)
Browse files Browse the repository at this point in the history
Signed-off-by: gran <[email protected]>
  • Loading branch information
gran-vmv authored Jul 30, 2024
1 parent 0dff0aa commit 0784c07
Show file tree
Hide file tree
Showing 16 changed files with 565 additions and 46 deletions.
8 changes: 7 additions & 1 deletion .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,4 +32,10 @@ linters:
- gosec
- goimports
- vet
- revive
- revive

issues:
exclude-rules:
- linters:
- staticcheck
text: "SA1019: lbs.RelaxScaleValidation"
6 changes: 3 additions & 3 deletions pkg/controllers/networkinfo/networkinfo_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
log.Error(err, "failed to check if namespace is shared", "Namespace", obj.GetNamespace())
return common.ResultRequeue, err
}
if !isShared {
if r.Service.NSXConfig.NsxConfig.UseAVILoadBalancer && !isShared {
err = r.Service.CreateOrUpdateAVIRule(createdVpc, obj.Namespace)
if err != nil {
state := &v1alpha1.VPCState{
Expand Down Expand Up @@ -116,7 +116,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
// if lb vpc enabled, read avi subnet path and cidr
// nsx bug, if set LoadBalancerVpcEndpoint.Enabled to false, when read this vpc back,
// LoadBalancerVpcEndpoint.Enabled will become a nil pointer.
if createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled {
if r.Service.NSXConfig.NsxConfig.UseAVILoadBalancer && createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled {
path, cidr, err = r.Service.GetAVISubnetInfo(*createdVpc)
if err != nil {
log.Error(err, "failed to read lb subnet path and cidr", "VPC", createdVpc.Id)
Expand All @@ -139,7 +139,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
LoadBalancerIPAddresses: cidr,
PrivateIPv4CIDRs: nc.PrivateIPv4CIDRs,
}
updateSuccess(r, &ctx, obj, r.Client, state, nc.Name, path)
updateSuccess(r, &ctx, obj, r.Client, state, nc.Name, path, r.Service.GetNSXLBSPath(*createdVpc.Id))
} else {
if controllerutil.ContainsFinalizer(obj, commonservice.NetworkInfoFinalizerName) {
metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeNetworkInfo)
Expand Down
11 changes: 6 additions & 5 deletions pkg/controllers/networkinfo/networkinfo_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,10 @@ func updateFail(r *NetworkInfoReconciler, c *context.Context, o *v1alpha1.Networ
}

func updateSuccess(r *NetworkInfoReconciler, c *context.Context, o *v1alpha1.NetworkInfo, client client.Client,
vpcState *v1alpha1.VPCState, ncName string, subnetPath string) {
vpcState *v1alpha1.VPCState, ncName string, subnetPath string, nsxLBSPath string) {
setNetworkInfoVPCStatus(c, o, client, vpcState)
// ako needs to know the avi subnet path created by nsx
setVPCNetworkConfigurationStatus(c, client, ncName, vpcState.Name, subnetPath)
setVPCNetworkConfigurationStatus(c, client, ncName, vpcState.Name, subnetPath, nsxLBSPath)
r.Recorder.Event(o, v1.EventTypeNormal, common.ReasonSuccessfulUpdate, "NetworkInfo CR has been successfully updated")
metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateSuccessTotal, common.MetricResTypeNetworkInfo)
}
Expand All @@ -59,16 +59,17 @@ func setNetworkInfoVPCStatus(ctx *context.Context, networkInfo *v1alpha1.Network
}
}

func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string) {
func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string, nsxLBSPath string) {
// read v1alpha1.VPCNetworkConfiguration by ncName
nc := &v1alpha1.VPCNetworkConfiguration{}
err := client.Get(*ctx, apitypes.NamespacedName{Name: ncName}, nc)
if err != nil {
log.Error(err, "failed to get VPCNetworkConfiguration", "Name", ncName)
}
createdVPCInfo := &v1alpha1.VPCInfo{
Name: vpcName,
AVISESubnetPath: aviSubnetPath,
Name: vpcName,
AVISESubnetPath: aviSubnetPath,
NSXLoadBalancerPath: nsxLBSPath,
}
// iterate through VPCNetworkConfiguration.Status.VPCs, if vpcName already exists, update it
for i, vpc := range nc.Status.VPCs {
Expand Down
3 changes: 3 additions & 0 deletions pkg/nsx/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ type Client struct {
SubnetsClient vpcs.SubnetsClient
RealizedStateClient realized_state.RealizedEntitiesClient
IPAddressAllocationClient vpcs.IpAddressAllocationsClient
VPCLBSClient vpcs.VpcLbsClient

NSXChecker NSXHealthChecker
NSXVerChecker NSXVersionChecker
Expand Down Expand Up @@ -165,6 +166,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client {
subnetStatusClient := subnets.NewStatusClient(restConnector(cluster))
realizedStateClient := realized_state.NewRealizedEntitiesClient(restConnector(cluster))
ipAddressAllocationClient := vpcs.NewIpAddressAllocationsClient(restConnector(cluster))
vpcLBSClient := vpcs.NewVpcLbsClient(restConnector(cluster))

vpcSecurityClient := vpcs.NewSecurityPoliciesClient(restConnector(cluster))
vpcRuleClient := vpc_sp.NewRulesClient(restConnector(cluster))
Expand Down Expand Up @@ -206,6 +208,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client {
SubnetStatusClient: subnetStatusClient,
VPCSecurityClient: vpcSecurityClient,
VPCRuleClient: vpcRuleClient,
VPCLBSClient: vpcLBSClient,

NSXChecker: *nsxChecker,
NSXVerChecker: *nsxVersionChecker,
Expand Down
3 changes: 3 additions & 0 deletions pkg/nsx/services/common/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ const (
TagScopeNCPVIFProjectUID string = "ncp/vif_project_uid"
TagScopeNCPPod string = "ncp/pod"
TagScopeNCPVNETInterface string = "ncp/vnet_interface"
TagScopeCreatedFor string = "nsx-op/created_for"
TagScopeVersion string = "nsx-op/version"
TagScopeCluster string = "nsx-op/cluster"
TagScopeNamespace string = "nsx-op/namespace"
Expand Down Expand Up @@ -74,6 +75,7 @@ const (
TagValueGroupSource string = "source"
TagValueGroupDestination string = "destination"
TagValueGroupAvi string = "avi"
TagValueSLB string = "SLB"
AnnotationVPCNetworkConfig string = "nsx.vmware.com/vpc_network_config"
AnnotationSharedVPCNamespace string = "nsx.vmware.com/shared_vpc_namespace"
AnnotationDefaultNetworkConfig string = "nsx.vmware.com/default"
Expand Down Expand Up @@ -147,6 +149,7 @@ var (
ResourceTypeVpc = "Vpc"
ResourceTypeSubnetPort = "VpcSubnetPort"
ResourceTypeVirtualMachine = "VirtualMachine"
ResourceTypeLBService = "LBService"
ResourceTypeShare = "Share"
ResourceTypeSharedResource = "SharedResource"
ResourceTypeChildSharedResource = "ChildSharedResource"
Expand Down
84 changes: 84 additions & 0 deletions pkg/nsx/services/common/wrap.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
package common

import (
"github.com/openlyinc/pointy"
"github.com/vmware/vsphere-automation-sdk-go/runtime/data"
"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model"
)

// WrapInfra TODO(gran) refactor existing code in other package
func (service *Service) WrapInfra(children []*data.StructValue) (*model.Infra, error) {
// This is the outermost layer of the hierarchy infra client.
// It doesn't need ID field.
resourceType := ResourceTypeInfra
infraObj := model.Infra{
Children: children,
ResourceType: &resourceType,
}
return &infraObj, nil
}

func (service *Service) WrapOrgRoot(children []*data.StructValue) (*model.OrgRoot, error) {
resourceType := ResourceTypeOrgRoot
orgRootObj := model.OrgRoot{
Children: children,
ResourceType: &resourceType,
}
return &orgRootObj, nil
}

func (service *Service) WrapOrg(org string, children []*data.StructValue) ([]*data.StructValue, error) {
targetType := ResourceTypeOrg
return wrapChildResourceReference(targetType, org, children)
}

func (service *Service) WrapProject(nsxtProject string, children []*data.StructValue) ([]*data.StructValue, error) {
targetType := ResourceTypeProject
return wrapChildResourceReference(targetType, nsxtProject, children)
}

func wrapChildResourceReference(targetType, id string, children []*data.StructValue) ([]*data.StructValue, error) {
resourceType := ResourceTypeChildResourceReference
childProject := model.ChildResourceReference{
Id: &id,
ResourceType: resourceType,
TargetType: &targetType,
Children: children,
}
dataValue, errors := NewConverter().ConvertToVapi(childProject, childProject.GetType__())
if len(errors) > 0 {
return nil, errors[0]
}
return []*data.StructValue{dataValue.(*data.StructValue)}, nil

}

func (service *Service) WrapVPC(vpc *model.Vpc) ([]*data.StructValue, error) {
vpc.ResourceType = pointy.String(ResourceTypeVpc)
childVpc := model.ChildVpc{
Id: vpc.Id,
MarkedForDelete: vpc.MarkedForDelete,
ResourceType: "ChildVpc",
Vpc: vpc,
}
dataValue, errs := NewConverter().ConvertToVapi(childVpc, childVpc.GetType__())
if len(errs) > 0 {
return nil, errs[0]
}
return []*data.StructValue{dataValue.(*data.StructValue)}, nil
}

func (service *Service) WrapLBS(lbs *model.LBService) ([]*data.StructValue, error) {
lbs.ResourceType = pointy.String(ResourceTypeLBService)
childLBService := model.ChildLBService{
Id: lbs.Id,
MarkedForDelete: lbs.MarkedForDelete,
ResourceType: "ChildLBService",
LbService: lbs,
}
dataValue, errs := NewConverter().ConvertToVapi(childLBService, childLBService.GetType__())
if len(errs) > 0 {
return nil, errs[0]
}
return []*data.StructValue{dataValue.(*data.StructValue)}, nil
}
2 changes: 1 addition & 1 deletion pkg/nsx/services/realizestate/realize_state.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func (service *RealizeStateService) CheckRealizeState(backoff wait.Backoff, inte
return err
}
for _, result := range results.Results {
if *result.EntityType != entityType {
if entityType != "" && result.EntityType != nil && *result.EntityType != entityType {
continue
}
if *result.State == model.GenericPolicyRealizedResource_STATE_REALIZED {
Expand Down
27 changes: 24 additions & 3 deletions pkg/nsx/services/vpc/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func buildPrivateIpBlock(networkInfo *v1alpha1.NetworkInfo, nsObj *v1.Namespace,
}

func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNetworkConfigInfo, cluster string, pathMap map[string]string,
nsxVPC *model.Vpc) (*model.Vpc,
nsxVPC *model.Vpc, useAVILB bool) (*model.Vpc,
error) {
vpc := &model.Vpc{}
if nsxVPC != nil {
Expand All @@ -61,7 +61,7 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe
return nil, nil
}
// for updating vpc case, use current vpc id, name
vpc = nsxVPC
*vpc = *nsxVPC
} else {
// for creating vpc case, fill in vpc properties based on networkconfig
vpcName := util.GenerateDisplayName("", "vpc", obj.GetNamespace(), "", cluster)
Expand All @@ -76,7 +76,10 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe
},
}
vpc.SiteInfos = siteInfos
vpc.LoadBalancerVpcEndpoint = &model.LoadBalancerVPCEndpoint{Enabled: &DefaultLoadBalancerVPCEndpointEnabled}
if useAVILB {
loadBalancerVPCEndpointEnabled := true
vpc.LoadBalancerVpcEndpoint = &model.LoadBalancerVPCEndpoint{Enabled: &loadBalancerVPCEndpointEnabled}
}
vpc.Tags = util.BuildBasicTags(cluster, obj, nsObj.UID)
}

Expand All @@ -89,3 +92,21 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe

return vpc, nil
}

func buildNSXLBS(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, cluster, lbsSize, vpcPath string, relaxScaleValidation *bool) (*model.LBService, error) {
lbs := &model.LBService{}
lbsName := util.GenerateDisplayName("", "vpc", nsObj.GetName(), "", cluster)
// Use VPC id for auto-created LBS id
lbs.Id = common.String(string(nsObj.GetUID()))
lbs.DisplayName = &lbsName
lbs.Tags = util.BuildBasicTags(cluster, obj, nsObj.GetUID())
// "created_for" is required by NCP, and "lb_t1_link_ip" is not needed for VPC
lbs.Tags = append(lbs.Tags, model.Tag{
Scope: common.String(common.TagScopeCreatedFor),
Tag: common.String(common.TagValueSLB),
})
lbs.Size = &lbsSize
lbs.ConnectivityPath = &vpcPath
lbs.RelaxScaleValidation = relaxScaleValidation
return lbs, nil
}
79 changes: 79 additions & 0 deletions pkg/nsx/services/vpc/builder_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
package vpc

import (
"fmt"
"strings"
"testing"

"github.com/stretchr/testify/assert"
"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
)

func Test_buildNSXLBS(t *testing.T) {
type args struct {
obj *v1alpha1.NetworkInfo
nsObj *v1.Namespace
cluster string
lbsSize string
vpcPath string
relaxScaleValidation *bool
}
tests := []struct {
name string
args args
want *model.LBService
wantErr assert.ErrorAssertionFunc
}{
{
name: "1",
args: args{
obj: &v1alpha1.NetworkInfo{
ObjectMeta: metav1.ObjectMeta{Namespace: "ns1"},
VPCs: nil,
},
nsObj: &v1.Namespace{
ObjectMeta: metav1.ObjectMeta{Name: "ns1", UID: "nsuid1"},
},
cluster: "cluster1",
lbsSize: model.LBService_SIZE_SMALL,
vpcPath: "/vpc1",
relaxScaleValidation: nil,
},
want: &model.LBService{
Id: common.String("nsuid1"),
DisplayName: common.String("vpc-cluster1--ns1"),
Tags: []model.Tag{
{
Scope: common.String(common.TagScopeCluster),
Tag: common.String("cluster1"),
},
{
Scope: common.String(common.TagScopeVersion),
Tag: common.String(strings.Join(common.TagValueVersion, ".")),
},
{Scope: common.String(common.TagScopeNamespace), Tag: common.String("ns1")},
{Scope: common.String(common.TagScopeNamespaceUID), Tag: common.String("nsuid1")},
{Scope: common.String(common.TagScopeCreatedFor), Tag: common.String(common.TagValueSLB)},
},
Size: common.String(model.LBService_SIZE_SMALL),
ConnectivityPath: common.String("/vpc1"),
RelaxScaleValidation: nil,
},
wantErr: assert.NoError,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := buildNSXLBS(tt.args.obj, tt.args.nsObj, tt.args.cluster, tt.args.lbsSize, tt.args.vpcPath, tt.args.relaxScaleValidation)
if !tt.wantErr(t, err, fmt.Sprintf("buildNSXLBS(%v, %v, %v, %v, %v, %v)", tt.args.obj, tt.args.nsObj, tt.args.cluster, tt.args.lbsSize, tt.args.vpcPath, tt.args.relaxScaleValidation)) {
return
}
assert.Equalf(t, tt.want, got, "buildNSXLBS(%v, %v, %v, %v, %v, %v)", tt.args.obj, tt.args.nsObj, tt.args.cluster, tt.args.lbsSize, tt.args.vpcPath, tt.args.relaxScaleValidation)
})
}
}
Loading

0 comments on commit 0784c07

Please sign in to comment.