Skip to content

Commit 0784c07

Browse files
gran-vmvgran-vmv
authored
[VPC] Support NSXLB for VPC (vmware-tanzu#618)
Signed-off-by: gran <[email protected]>
1 parent 0dff0aa commit 0784c07

File tree

16 files changed

+565
-46
lines changed

16 files changed

+565
-46
lines changed

‎.golangci.yml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,4 +32,10 @@ linters:
3232
- gosec
3333
- goimports
3434
- vet
35-
- revive
35+
- revive
36+
37+
issues:
38+
exclude-rules:
39+
- linters:
40+
- staticcheck
41+
text: "SA1019: lbs.RelaxScaleValidation"

‎pkg/controllers/networkinfo/networkinfo_controller.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
7878
log.Error(err, "failed to check if namespace is shared", "Namespace", obj.GetNamespace())
7979
return common.ResultRequeue, err
8080
}
81-
if !isShared {
81+
if r.Service.NSXConfig.NsxConfig.UseAVILoadBalancer && !isShared {
8282
err = r.Service.CreateOrUpdateAVIRule(createdVpc, obj.Namespace)
8383
if err != nil {
8484
state := &v1alpha1.VPCState{
@@ -116,7 +116,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
116116
// if lb vpc enabled, read avi subnet path and cidr
117117
// nsx bug, if set LoadBalancerVpcEndpoint.Enabled to false, when read this vpc back,
118118
// LoadBalancerVpcEndpoint.Enabled will become a nil pointer.
119-
if createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled {
119+
if r.Service.NSXConfig.NsxConfig.UseAVILoadBalancer && createdVpc.LoadBalancerVpcEndpoint.Enabled != nil && *createdVpc.LoadBalancerVpcEndpoint.Enabled {
120120
path, cidr, err = r.Service.GetAVISubnetInfo(*createdVpc)
121121
if err != nil {
122122
log.Error(err, "failed to read lb subnet path and cidr", "VPC", createdVpc.Id)
@@ -139,7 +139,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
139139
LoadBalancerIPAddresses: cidr,
140140
PrivateIPv4CIDRs: nc.PrivateIPv4CIDRs,
141141
}
142-
updateSuccess(r, &ctx, obj, r.Client, state, nc.Name, path)
142+
updateSuccess(r, &ctx, obj, r.Client, state, nc.Name, path, r.Service.GetNSXLBSPath(*createdVpc.Id))
143143
} else {
144144
if controllerutil.ContainsFinalizer(obj, commonservice.NetworkInfoFinalizerName) {
145145
metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeNetworkInfo)

‎pkg/controllers/networkinfo/networkinfo_utils.go

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,10 @@ func updateFail(r *NetworkInfoReconciler, c *context.Context, o *v1alpha1.Networ
2929
}
3030

3131
func updateSuccess(r *NetworkInfoReconciler, c *context.Context, o *v1alpha1.NetworkInfo, client client.Client,
32-
vpcState *v1alpha1.VPCState, ncName string, subnetPath string) {
32+
vpcState *v1alpha1.VPCState, ncName string, subnetPath string, nsxLBSPath string) {
3333
setNetworkInfoVPCStatus(c, o, client, vpcState)
3434
// ako needs to know the avi subnet path created by nsx
35-
setVPCNetworkConfigurationStatus(c, client, ncName, vpcState.Name, subnetPath)
35+
setVPCNetworkConfigurationStatus(c, client, ncName, vpcState.Name, subnetPath, nsxLBSPath)
3636
r.Recorder.Event(o, v1.EventTypeNormal, common.ReasonSuccessfulUpdate, "NetworkInfo CR has been successfully updated")
3737
metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateSuccessTotal, common.MetricResTypeNetworkInfo)
3838
}
@@ -59,16 +59,17 @@ func setNetworkInfoVPCStatus(ctx *context.Context, networkInfo *v1alpha1.Network
5959
}
6060
}
6161

62-
func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string) {
62+
func setVPCNetworkConfigurationStatus(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string, nsxLBSPath string) {
6363
// read v1alpha1.VPCNetworkConfiguration by ncName
6464
nc := &v1alpha1.VPCNetworkConfiguration{}
6565
err := client.Get(*ctx, apitypes.NamespacedName{Name: ncName}, nc)
6666
if err != nil {
6767
log.Error(err, "failed to get VPCNetworkConfiguration", "Name", ncName)
6868
}
6969
createdVPCInfo := &v1alpha1.VPCInfo{
70-
Name: vpcName,
71-
AVISESubnetPath: aviSubnetPath,
70+
Name: vpcName,
71+
AVISESubnetPath: aviSubnetPath,
72+
NSXLoadBalancerPath: nsxLBSPath,
7273
}
7374
// iterate through VPCNetworkConfiguration.Status.VPCs, if vpcName already exists, update it
7475
for i, vpc := range nc.Status.VPCs {

‎pkg/nsx/client.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ type Client struct {
8585
SubnetsClient vpcs.SubnetsClient
8686
RealizedStateClient realized_state.RealizedEntitiesClient
8787
IPAddressAllocationClient vpcs.IpAddressAllocationsClient
88+
VPCLBSClient vpcs.VpcLbsClient
8889

8990
NSXChecker NSXHealthChecker
9091
NSXVerChecker NSXVersionChecker
@@ -165,6 +166,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client {
165166
subnetStatusClient := subnets.NewStatusClient(restConnector(cluster))
166167
realizedStateClient := realized_state.NewRealizedEntitiesClient(restConnector(cluster))
167168
ipAddressAllocationClient := vpcs.NewIpAddressAllocationsClient(restConnector(cluster))
169+
vpcLBSClient := vpcs.NewVpcLbsClient(restConnector(cluster))
168170

169171
vpcSecurityClient := vpcs.NewSecurityPoliciesClient(restConnector(cluster))
170172
vpcRuleClient := vpc_sp.NewRulesClient(restConnector(cluster))
@@ -206,6 +208,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client {
206208
SubnetStatusClient: subnetStatusClient,
207209
VPCSecurityClient: vpcSecurityClient,
208210
VPCRuleClient: vpcRuleClient,
211+
VPCLBSClient: vpcLBSClient,
209212

210213
NSXChecker: *nsxChecker,
211214
NSXVerChecker: *nsxVersionChecker,

‎pkg/nsx/services/common/types.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ const (
2828
TagScopeNCPVIFProjectUID string = "ncp/vif_project_uid"
2929
TagScopeNCPPod string = "ncp/pod"
3030
TagScopeNCPVNETInterface string = "ncp/vnet_interface"
31+
TagScopeCreatedFor string = "nsx-op/created_for"
3132
TagScopeVersion string = "nsx-op/version"
3233
TagScopeCluster string = "nsx-op/cluster"
3334
TagScopeNamespace string = "nsx-op/namespace"
@@ -74,6 +75,7 @@ const (
7475
TagValueGroupSource string = "source"
7576
TagValueGroupDestination string = "destination"
7677
TagValueGroupAvi string = "avi"
78+
TagValueSLB string = "SLB"
7779
AnnotationVPCNetworkConfig string = "nsx.vmware.com/vpc_network_config"
7880
AnnotationSharedVPCNamespace string = "nsx.vmware.com/shared_vpc_namespace"
7981
AnnotationDefaultNetworkConfig string = "nsx.vmware.com/default"
@@ -147,6 +149,7 @@ var (
147149
ResourceTypeVpc = "Vpc"
148150
ResourceTypeSubnetPort = "VpcSubnetPort"
149151
ResourceTypeVirtualMachine = "VirtualMachine"
152+
ResourceTypeLBService = "LBService"
150153
ResourceTypeShare = "Share"
151154
ResourceTypeSharedResource = "SharedResource"
152155
ResourceTypeChildSharedResource = "ChildSharedResource"

‎pkg/nsx/services/common/wrap.go

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
package common
2+
3+
import (
4+
"github.com/openlyinc/pointy"
5+
"github.com/vmware/vsphere-automation-sdk-go/runtime/data"
6+
"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model"
7+
)
8+
9+
// WrapInfra TODO(gran) refactor existing code in other package
10+
func (service *Service) WrapInfra(children []*data.StructValue) (*model.Infra, error) {
11+
// This is the outermost layer of the hierarchy infra client.
12+
// It doesn't need ID field.
13+
resourceType := ResourceTypeInfra
14+
infraObj := model.Infra{
15+
Children: children,
16+
ResourceType: &resourceType,
17+
}
18+
return &infraObj, nil
19+
}
20+
21+
func (service *Service) WrapOrgRoot(children []*data.StructValue) (*model.OrgRoot, error) {
22+
resourceType := ResourceTypeOrgRoot
23+
orgRootObj := model.OrgRoot{
24+
Children: children,
25+
ResourceType: &resourceType,
26+
}
27+
return &orgRootObj, nil
28+
}
29+
30+
func (service *Service) WrapOrg(org string, children []*data.StructValue) ([]*data.StructValue, error) {
31+
targetType := ResourceTypeOrg
32+
return wrapChildResourceReference(targetType, org, children)
33+
}
34+
35+
func (service *Service) WrapProject(nsxtProject string, children []*data.StructValue) ([]*data.StructValue, error) {
36+
targetType := ResourceTypeProject
37+
return wrapChildResourceReference(targetType, nsxtProject, children)
38+
}
39+
40+
func wrapChildResourceReference(targetType, id string, children []*data.StructValue) ([]*data.StructValue, error) {
41+
resourceType := ResourceTypeChildResourceReference
42+
childProject := model.ChildResourceReference{
43+
Id: &id,
44+
ResourceType: resourceType,
45+
TargetType: &targetType,
46+
Children: children,
47+
}
48+
dataValue, errors := NewConverter().ConvertToVapi(childProject, childProject.GetType__())
49+
if len(errors) > 0 {
50+
return nil, errors[0]
51+
}
52+
return []*data.StructValue{dataValue.(*data.StructValue)}, nil
53+
54+
}
55+
56+
func (service *Service) WrapVPC(vpc *model.Vpc) ([]*data.StructValue, error) {
57+
vpc.ResourceType = pointy.String(ResourceTypeVpc)
58+
childVpc := model.ChildVpc{
59+
Id: vpc.Id,
60+
MarkedForDelete: vpc.MarkedForDelete,
61+
ResourceType: "ChildVpc",
62+
Vpc: vpc,
63+
}
64+
dataValue, errs := NewConverter().ConvertToVapi(childVpc, childVpc.GetType__())
65+
if len(errs) > 0 {
66+
return nil, errs[0]
67+
}
68+
return []*data.StructValue{dataValue.(*data.StructValue)}, nil
69+
}
70+
71+
func (service *Service) WrapLBS(lbs *model.LBService) ([]*data.StructValue, error) {
72+
lbs.ResourceType = pointy.String(ResourceTypeLBService)
73+
childLBService := model.ChildLBService{
74+
Id: lbs.Id,
75+
MarkedForDelete: lbs.MarkedForDelete,
76+
ResourceType: "ChildLBService",
77+
LbService: lbs,
78+
}
79+
dataValue, errs := NewConverter().ConvertToVapi(childLBService, childLBService.GetType__())
80+
if len(errs) > 0 {
81+
return nil, errs[0]
82+
}
83+
return []*data.StructValue{dataValue.(*data.StructValue)}, nil
84+
}

‎pkg/nsx/services/realizestate/realize_state.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ func (service *RealizeStateService) CheckRealizeState(backoff wait.Backoff, inte
4949
return err
5050
}
5151
for _, result := range results.Results {
52-
if *result.EntityType != entityType {
52+
if entityType != "" && result.EntityType != nil && *result.EntityType != entityType {
5353
continue
5454
}
5555
if *result.State == model.GenericPolicyRealizedResource_STATE_REALIZED {

‎pkg/nsx/services/vpc/builder.go

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ func buildPrivateIpBlock(networkInfo *v1alpha1.NetworkInfo, nsObj *v1.Namespace,
5151
}
5252

5353
func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNetworkConfigInfo, cluster string, pathMap map[string]string,
54-
nsxVPC *model.Vpc) (*model.Vpc,
54+
nsxVPC *model.Vpc, useAVILB bool) (*model.Vpc,
5555
error) {
5656
vpc := &model.Vpc{}
5757
if nsxVPC != nil {
@@ -61,7 +61,7 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe
6161
return nil, nil
6262
}
6363
// for updating vpc case, use current vpc id, name
64-
vpc = nsxVPC
64+
*vpc = *nsxVPC
6565
} else {
6666
// for creating vpc case, fill in vpc properties based on networkconfig
6767
vpcName := util.GenerateDisplayName("", "vpc", obj.GetNamespace(), "", cluster)
@@ -76,7 +76,10 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe
7676
},
7777
}
7878
vpc.SiteInfos = siteInfos
79-
vpc.LoadBalancerVpcEndpoint = &model.LoadBalancerVPCEndpoint{Enabled: &DefaultLoadBalancerVPCEndpointEnabled}
79+
if useAVILB {
80+
loadBalancerVPCEndpointEnabled := true
81+
vpc.LoadBalancerVpcEndpoint = &model.LoadBalancerVPCEndpoint{Enabled: &loadBalancerVPCEndpointEnabled}
82+
}
8083
vpc.Tags = util.BuildBasicTags(cluster, obj, nsObj.UID)
8184
}
8285

@@ -89,3 +92,21 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe
8992

9093
return vpc, nil
9194
}
95+
96+
func buildNSXLBS(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, cluster, lbsSize, vpcPath string, relaxScaleValidation *bool) (*model.LBService, error) {
97+
lbs := &model.LBService{}
98+
lbsName := util.GenerateDisplayName("", "vpc", nsObj.GetName(), "", cluster)
99+
// Use VPC id for auto-created LBS id
100+
lbs.Id = common.String(string(nsObj.GetUID()))
101+
lbs.DisplayName = &lbsName
102+
lbs.Tags = util.BuildBasicTags(cluster, obj, nsObj.GetUID())
103+
// "created_for" is required by NCP, and "lb_t1_link_ip" is not needed for VPC
104+
lbs.Tags = append(lbs.Tags, model.Tag{
105+
Scope: common.String(common.TagScopeCreatedFor),
106+
Tag: common.String(common.TagValueSLB),
107+
})
108+
lbs.Size = &lbsSize
109+
lbs.ConnectivityPath = &vpcPath
110+
lbs.RelaxScaleValidation = relaxScaleValidation
111+
return lbs, nil
112+
}

‎pkg/nsx/services/vpc/builder_test.go

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
package vpc
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
"testing"
7+
8+
"github.com/stretchr/testify/assert"
9+
"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model"
10+
v1 "k8s.io/api/core/v1"
11+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
12+
13+
"github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1"
14+
"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
15+
)
16+
17+
func Test_buildNSXLBS(t *testing.T) {
18+
type args struct {
19+
obj *v1alpha1.NetworkInfo
20+
nsObj *v1.Namespace
21+
cluster string
22+
lbsSize string
23+
vpcPath string
24+
relaxScaleValidation *bool
25+
}
26+
tests := []struct {
27+
name string
28+
args args
29+
want *model.LBService
30+
wantErr assert.ErrorAssertionFunc
31+
}{
32+
{
33+
name: "1",
34+
args: args{
35+
obj: &v1alpha1.NetworkInfo{
36+
ObjectMeta: metav1.ObjectMeta{Namespace: "ns1"},
37+
VPCs: nil,
38+
},
39+
nsObj: &v1.Namespace{
40+
ObjectMeta: metav1.ObjectMeta{Name: "ns1", UID: "nsuid1"},
41+
},
42+
cluster: "cluster1",
43+
lbsSize: model.LBService_SIZE_SMALL,
44+
vpcPath: "/vpc1",
45+
relaxScaleValidation: nil,
46+
},
47+
want: &model.LBService{
48+
Id: common.String("nsuid1"),
49+
DisplayName: common.String("vpc-cluster1--ns1"),
50+
Tags: []model.Tag{
51+
{
52+
Scope: common.String(common.TagScopeCluster),
53+
Tag: common.String("cluster1"),
54+
},
55+
{
56+
Scope: common.String(common.TagScopeVersion),
57+
Tag: common.String(strings.Join(common.TagValueVersion, ".")),
58+
},
59+
{Scope: common.String(common.TagScopeNamespace), Tag: common.String("ns1")},
60+
{Scope: common.String(common.TagScopeNamespaceUID), Tag: common.String("nsuid1")},
61+
{Scope: common.String(common.TagScopeCreatedFor), Tag: common.String(common.TagValueSLB)},
62+
},
63+
Size: common.String(model.LBService_SIZE_SMALL),
64+
ConnectivityPath: common.String("/vpc1"),
65+
RelaxScaleValidation: nil,
66+
},
67+
wantErr: assert.NoError,
68+
},
69+
}
70+
for _, tt := range tests {
71+
t.Run(tt.name, func(t *testing.T) {
72+
got, err := buildNSXLBS(tt.args.obj, tt.args.nsObj, tt.args.cluster, tt.args.lbsSize, tt.args.vpcPath, tt.args.relaxScaleValidation)
73+
if !tt.wantErr(t, err, fmt.Sprintf("buildNSXLBS(%v, %v, %v, %v, %v, %v)", tt.args.obj, tt.args.nsObj, tt.args.cluster, tt.args.lbsSize, tt.args.vpcPath, tt.args.relaxScaleValidation)) {
74+
return
75+
}
76+
assert.Equalf(t, tt.want, got, "buildNSXLBS(%v, %v, %v, %v, %v, %v)", tt.args.obj, tt.args.nsObj, tt.args.cluster, tt.args.lbsSize, tt.args.vpcPath, tt.args.relaxScaleValidation)
77+
})
78+
}
79+
}

0 commit comments

Comments
 (0)