From 58cc17c277481247bd3e52d7fba41b391ceae5b3 Mon Sep 17 00:00:00 2001 From: Gregory Thiemonge Date: Fri, 7 Jun 2024 07:21:57 +0000 Subject: [PATCH 1/6] Create provider network based on NAD Config Configuration is based on the NAD passed in the Octavia.Spec.OctaviaNetworkAttachment string (default 'octavia') JIRA: OSPRH-7553 --- api/bases/octavia.openstack.org_octavias.yaml | 6 ++ api/v1beta1/octavia_types.go | 5 ++ .../bases/octavia.openstack.org_octavias.yaml | 6 ++ controllers/octavia_controller.go | 11 +++ pkg/octavia/lb_mgmt_network.go | 40 ++++++---- pkg/octavia/network_consts.go | 18 +---- pkg/octavia/network_parameters.go | 80 +++++++++++++++++++ 7 files changed, 137 insertions(+), 29 deletions(-) create mode 100644 pkg/octavia/network_parameters.go diff --git a/api/bases/octavia.openstack.org_octavias.yaml b/api/bases/octavia.openstack.org_octavias.yaml index 0cce1775..dc5766c5 100644 --- a/api/bases/octavia.openstack.org_octavias.yaml +++ b/api/bases/octavia.openstack.org_octavias.yaml @@ -892,6 +892,11 @@ spec: - secret - serviceAccount type: object + octaviaNetworkAttachment: + default: octavia + description: OctaviaNetworkAttachment is a NetworkAttachment resource + name for the Octavia Management Network + type: string octaviaWorker: description: OctaviaHousekeeping - Spec definition for the Octavia Housekeeping agent for the Octavia deployment @@ -1197,6 +1202,7 @@ spec: - apacheContainerImage - databaseInstance - octaviaAPI + - octaviaNetworkAttachment - rabbitMqClusterName - secret type: object diff --git a/api/v1beta1/octavia_types.go b/api/v1beta1/octavia_types.go index ff6dc37c..1dd4d2b5 100644 --- a/api/v1beta1/octavia_types.go +++ b/api/v1beta1/octavia_types.go @@ -192,6 +192,11 @@ type OctaviaSpecBase struct { // +kubebuilder:validation:Required // Apache Container Image URL ApacheContainerImage string `json:"apacheContainerImage"` + + // +kubebuilder:validation:Required + // +kubebuilder:default=octavia + // OctaviaNetworkAttachment is a NetworkAttachment resource name for the Octavia Management Network + OctaviaNetworkAttachment string `json:"octaviaNetworkAttachment"` } // PasswordSelector to identify the DB and AdminUser password from the Secret diff --git a/config/crd/bases/octavia.openstack.org_octavias.yaml b/config/crd/bases/octavia.openstack.org_octavias.yaml index 0cce1775..dc5766c5 100644 --- a/config/crd/bases/octavia.openstack.org_octavias.yaml +++ b/config/crd/bases/octavia.openstack.org_octavias.yaml @@ -892,6 +892,11 @@ spec: - secret - serviceAccount type: object + octaviaNetworkAttachment: + default: octavia + description: OctaviaNetworkAttachment is a NetworkAttachment resource + name for the Octavia Management Network + type: string octaviaWorker: description: OctaviaHousekeeping - Spec definition for the Octavia Housekeeping agent for the Octavia deployment @@ -1197,6 +1202,7 @@ spec: - apacheContainerImage - databaseInstance - octaviaAPI + - octaviaNetworkAttachment - rabbitMqClusterName - secret type: object diff --git a/controllers/octavia_controller.go b/controllers/octavia_controller.go index e780c740..74dd5423 100644 --- a/controllers/octavia_controller.go +++ b/controllers/octavia_controller.go @@ -642,6 +642,16 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav // Amphora reconciliation // ------------------------------------------------------------------------------------------------------------ + nad, err := nad.GetNADWithName(ctx, helper, instance.Spec.OctaviaNetworkAttachment, instance.Namespace) + if err != nil { + return ctrl.Result{}, err + } + + networkParameters, err := octavia.GetNetworkParametersFromNAD(nad) + if err != nil { + return ctrl.Result{}, err + } + // Create load balancer management network and get its Id (networkInfo is actually a struct and contains // multiple details. networkInfo, err := octavia.EnsureAmphoraManagementNetwork( @@ -649,6 +659,7 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav instance.Namespace, instance.Spec.TenantName, &instance.Spec.LbMgmtNetworks, + networkParameters, &Log, helper, ) diff --git a/pkg/octavia/lb_mgmt_network.go b/pkg/octavia/lb_mgmt_network.go index 1e2fb698..83a875c8 100644 --- a/pkg/octavia/lb_mgmt_network.go +++ b/pkg/octavia/lb_mgmt_network.go @@ -291,20 +291,29 @@ func ensureNetworkExt(client *gophercloud.ServiceClient, createOpts networks.Cre return foundNetwork, nil } -func ensureProvSubnet(client *gophercloud.ServiceClient, providerNetwork *networks.Network, log *logr.Logger) ( - *subnets.Subnet, error) { - gatewayIP := LbProvSubnetGatewayIP +func ensureProvSubnet( + client *gophercloud.ServiceClient, + providerNetwork *networks.Network, + networkParameters *NetworkParameters, + log *logr.Logger, +) (*subnets.Subnet, error) { + var gatewayIP string + if networkParameters.Gateway.IsValid() { + gatewayIP = networkParameters.Gateway.String() + } else { + gatewayIP = "" + } createOpts := subnets.CreateOpts{ Name: LbProvSubnetName, Description: LbProvSubnetDescription, NetworkID: providerNetwork.ID, TenantID: providerNetwork.TenantID, - CIDR: LbProvSubnetCIDR, + CIDR: networkParameters.CIDR.String(), IPVersion: gophercloud.IPVersion(4), AllocationPools: []subnets.AllocationPool{ { - Start: LbProvSubnetAllocationPoolStart, - End: LbProvSubnetAllocationPoolEnd, + Start: networkParameters.AllocationStart.String(), + End: networkParameters.AllocationEnd.String(), }, }, GatewayIP: &gatewayIP, @@ -339,6 +348,7 @@ func ensureLbMgmtSubnet( client *gophercloud.ServiceClient, networkDetails *octaviav1.OctaviaLbMgmtNetworks, tenantNetwork *networks.Network, + networkParameters *NetworkParameters, log *logr.Logger, ) (*subnets.Subnet, error) { ipVersion := networkDetails.SubnetIPVersion @@ -381,7 +391,7 @@ func ensureLbMgmtSubnet( }, HostRoutes: []subnets.HostRoute{ { - DestinationCIDR: LbProvSubnetCIDR, + DestinationCIDR: networkParameters.CIDR.String(), NextHop: LbMgmtRouterPortIPv4, }, }, @@ -422,10 +432,10 @@ func ensureLbMgmtNetwork(client *gophercloud.ServiceClient, networkDetails *octa return mgmtNetwork, nil } -func externalFixedIPs(subnetID string) []routers.ExternalFixedIP { +func externalFixedIPs(subnetID string, networkParameters *NetworkParameters) []routers.ExternalFixedIP { ips := []routers.ExternalFixedIP{ { - IPAddress: LbRouterFixedIPAddress, + IPAddress: networkParameters.RouterIPAddress.String(), SubnetID: subnetID, }, } @@ -452,6 +462,7 @@ func compareExternalFixedIPs(a []routers.ExternalFixedIP, b []routers.ExternalFi func reconcileRouter(client *gophercloud.ServiceClient, router *routers.Router, gatewayNetwork *networks.Network, gatewaySubnet *subnets.Subnet, + networkParameters *NetworkParameters, log *logr.Logger) (*routers.Router, error) { if !router.AdminStateUp { @@ -464,7 +475,7 @@ func reconcileRouter(client *gophercloud.ServiceClient, router *routers.Router, needsUpdate := false updateInfo := routers.UpdateOpts{} enableSNAT := false - fixedIPs := externalFixedIPs(gatewaySubnet.ID) + fixedIPs := externalFixedIPs(gatewaySubnet.ID, networkParameters) // // TODO(beagles) we don't care about the other fields right now because we @@ -752,6 +763,7 @@ func EnsureAmphoraManagementNetwork( ns string, tenantName string, netDetails *octaviav1.OctaviaLbMgmtNetworks, + networkParameters *NetworkParameters, log *logr.Logger, helper *helper.Helper, ) (NetworkProvisioningSummary, error) { @@ -772,7 +784,7 @@ func EnsureAmphoraManagementNetwork( if err != nil { return NetworkProvisioningSummary{}, err } - tenantSubnet, err := ensureLbMgmtSubnet(client, netDetails, tenantNetwork, log) + tenantSubnet, err := ensureLbMgmtSubnet(client, netDetails, tenantNetwork, networkParameters, log) if err != nil { return NetworkProvisioningSummary{}, err } @@ -811,7 +823,7 @@ func EnsureAmphoraManagementNetwork( return NetworkProvisioningSummary{}, err } - providerSubnet, err := ensureProvSubnet(client, providerNetwork, log) + providerSubnet, err := ensureProvSubnet(client, providerNetwork, networkParameters, log) if err != nil { return NetworkProvisioningSummary{}, err } @@ -822,7 +834,7 @@ func EnsureAmphoraManagementNetwork( } if router != nil { log.Info("Router object found, reconciling") - router, err = reconcileRouter(client, router, providerNetwork, providerSubnet, log) + router, err = reconcileRouter(client, router, providerNetwork, providerSubnet, networkParameters, log) if err != nil { return NetworkProvisioningSummary{}, err } @@ -833,7 +845,7 @@ func EnsureAmphoraManagementNetwork( gatewayInfo := routers.GatewayInfo{ NetworkID: providerNetwork.ID, EnableSNAT: &enableSNAT, - ExternalFixedIPs: externalFixedIPs(providerSubnet.ID), + ExternalFixedIPs: externalFixedIPs(providerSubnet.ID, networkParameters), } adminStateUp := true createOpts := routers.CreateOpts{ diff --git a/pkg/octavia/network_consts.go b/pkg/octavia/network_consts.go index d2c3c55e..73b97e7c 100644 --- a/pkg/octavia/network_consts.go +++ b/pkg/octavia/network_consts.go @@ -83,19 +83,10 @@ const ( // LbProvSubnetDescription - LbProvSubnetDescription = "LBaaS Management Provider Subnet" - // IPv4 consts - - // LbProvSubnetCIDR - - LbProvSubnetCIDR = "172.23.0.0/24" - - // LbProvSubnetAllocationPoolStart - - LbProvSubnetAllocationPoolStart = "172.23.0.100" + // LbProvSubnetPoolSize - + LbProvSubnetPoolSize = 25 - // LbProvSubnetAllocationPoolEnd - - LbProvSubnetAllocationPoolEnd = "172.23.0.125" - - // LbProvSubnetGatewayIP - - LbProvSubnetGatewayIP = "" + // IPv4 consts // TODO(beagles): support IPv6 for the provider network. // LbRouterName - @@ -104,9 +95,6 @@ const ( // LbProvPhysicalNet - LbProvPhysicalNet = "octavia" - // LbRouterFixedIPAddress - LbRouterFixedIPAddress = "172.23.0.150" - // LbMgmtRouterPortName LbMgmtRouterPortName = "lb-mgmt-router-port" diff --git a/pkg/octavia/network_parameters.go b/pkg/octavia/network_parameters.go new file mode 100644 index 00000000..292ff31b --- /dev/null +++ b/pkg/octavia/network_parameters.go @@ -0,0 +1,80 @@ +package octavia + +import ( + "encoding/json" + "fmt" + "net/netip" + + networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" +) + +// NetworkParameters - Parameters for the Octavia networks, based on the config of the NAD +type NetworkParameters struct { + CIDR netip.Prefix + AllocationStart netip.Addr + AllocationEnd netip.Addr + Gateway netip.Addr + RouterIPAddress netip.Addr +} + +// NADConfig - IPAM parameters of the NAD +type NADConfig struct { + IPAM NADIpam `json:"ipam"` +} + +type NADIpam struct { + CIDR netip.Prefix `json:"range"` + RangeStart netip.Addr `json:"range_start"` + RangeEnd netip.Addr `json:"range_end"` + + Routes []NADRoute `json:"routes"` +} + +type NADRoute struct { + Gateway netip.Addr `json:"gw"` +} + +func getConfigFromNAD( + nad *networkv1.NetworkAttachmentDefinition, +) (*NADConfig, error) { + nadConfig := &NADConfig{} + jsonDoc := []byte(nad.Spec.Config) + err := json.Unmarshal(jsonDoc, nadConfig) + if err != nil { + return nil, err + } + + return nadConfig, nil +} + +func GetNetworkParametersFromNAD( + nad *networkv1.NetworkAttachmentDefinition, +) (*NetworkParameters, error) { + networkParameters := &NetworkParameters{} + + nadConfig, err := getConfigFromNAD(nad) + if err != nil { + return nil, fmt.Errorf("cannot read network parameters: %w", err) + } + + networkParameters.CIDR = nadConfig.IPAM.CIDR + + networkParameters.AllocationStart = nadConfig.IPAM.RangeEnd.Next() + end := networkParameters.AllocationStart + for i := 0; i < LbProvSubnetPoolSize; i++ { + if !networkParameters.CIDR.Contains(end) { + return nil, fmt.Errorf("cannot allocate %d IP addresses in %s", LbProvSubnetPoolSize, networkParameters.CIDR) + } + end = end.Next() + } + networkParameters.AllocationEnd = end + // TODO(gthiemonge) Remove routes from NAD, manage them in the operator + if len(nadConfig.IPAM.Routes) > 0 { + networkParameters.RouterIPAddress = nadConfig.IPAM.Routes[0].Gateway + } else { + return nil, fmt.Errorf("cannot find gateway information in network attachment") + } + // Gateway is currently unset + + return networkParameters, err +} From d36c6df2a2539b63baf75e30c8fcf83e81c55b4f Mon Sep 17 00:00:00 2001 From: Gregory Thiemonge Date: Fri, 7 Jun 2024 14:06:04 +0000 Subject: [PATCH 2/6] Remove hardcoded values for the tenant network The parameters are computed by using the destination of the route of the NAD --- api/bases/octavia.openstack.org_octavias.yaml | 5 -- api/v1beta1/octavia_types.go | 7 +- .../bases/octavia.openstack.org_octavias.yaml | 5 -- pkg/octavia/lb_mgmt_network.go | 51 +++++------- pkg/octavia/network_consts.go | 28 ------- pkg/octavia/network_parameters.go | 83 +++++++++++++++---- 6 files changed, 92 insertions(+), 87 deletions(-) diff --git a/api/bases/octavia.openstack.org_octavias.yaml b/api/bases/octavia.openstack.org_octavias.yaml index dc5766c5..c83b7364 100644 --- a/api/bases/octavia.openstack.org_octavias.yaml +++ b/api/bases/octavia.openstack.org_octavias.yaml @@ -103,7 +103,6 @@ spec: lbMgmtNetwork: default: manageLbMgmtNetworks: true - subnetIpVersion: 4 description: OctaviaLbMgmtNetworks Settings for Octavia management networks properties: @@ -116,10 +115,6 @@ spec: manageLbMgmtNetworks: default: true type: boolean - subnetIpVersion: - default: 4 - description: IP Version of the managed subnets - type: integer type: object nodeSelector: additionalProperties: diff --git a/api/v1beta1/octavia_types.go b/api/v1beta1/octavia_types.go index 1dd4d2b5..10bf7980 100644 --- a/api/v1beta1/octavia_types.go +++ b/api/v1beta1/octavia_types.go @@ -160,7 +160,7 @@ type OctaviaSpecBase struct { TenantName string `json:"tenantName"` // +kubebuilder:validation:Optional - // +kubebuilder:default={manageLbMgmtNetworks: true, subnetIpVersion: 4} + // +kubebuilder:default={manageLbMgmtNetworks: true} LbMgmtNetworks OctaviaLbMgmtNetworks `json:"lbMgmtNetwork"` // +kubebuilder:validation:Optional @@ -218,11 +218,6 @@ type OctaviaLbMgmtNetworks struct { // +kubebuilder:default=true ManageLbMgmtNetworks bool `json:"manageLbMgmtNetworks,omitempty"` - // +kubebuilder:validation:Optional - // +kubebuilder:default=4 - // IP Version of the managed subnets - SubnetIPVersion int `json:"subnetIpVersion,omitempty"` - // +kubebuilder:validation:Optional // Availability zones for the octavia management network resources AvailabilityZones []string `json:"availabilityZones,omitempty"` diff --git a/config/crd/bases/octavia.openstack.org_octavias.yaml b/config/crd/bases/octavia.openstack.org_octavias.yaml index dc5766c5..c83b7364 100644 --- a/config/crd/bases/octavia.openstack.org_octavias.yaml +++ b/config/crd/bases/octavia.openstack.org_octavias.yaml @@ -103,7 +103,6 @@ spec: lbMgmtNetwork: default: manageLbMgmtNetworks: true - subnetIpVersion: 4 description: OctaviaLbMgmtNetworks Settings for Octavia management networks properties: @@ -116,10 +115,6 @@ spec: manageLbMgmtNetworks: default: true type: boolean - subnetIpVersion: - default: 4 - description: IP Version of the managed subnets - type: integer type: object nodeSelector: additionalProperties: diff --git a/pkg/octavia/lb_mgmt_network.go b/pkg/octavia/lb_mgmt_network.go index 83a875c8..87173688 100644 --- a/pkg/octavia/lb_mgmt_network.go +++ b/pkg/octavia/lb_mgmt_network.go @@ -72,12 +72,8 @@ func findPort(client *gophercloud.ServiceClient, networkID string, ipAddress str } func ensurePort(client *gophercloud.ServiceClient, tenantNetwork *networks.Network, tenantSubnet *subnets.Subnet, - securityGroups *[]string, log *logr.Logger) (*ports.Port, error) { - ipAddress := LbMgmtRouterPortIPv4 - if tenantSubnet.IPVersion == 6 { - ipAddress = LbMgmtRouterPortIPv6 - } - + securityGroups *[]string, networkParameters *NetworkParameters, log *logr.Logger) (*ports.Port, error) { + ipAddress := networkParameters.TenantGateway.String() p, err := findPort(client, tenantNetwork.ID, ipAddress, log) if err != nil { return nil, err @@ -297,23 +293,18 @@ func ensureProvSubnet( networkParameters *NetworkParameters, log *logr.Logger, ) (*subnets.Subnet, error) { - var gatewayIP string - if networkParameters.Gateway.IsValid() { - gatewayIP = networkParameters.Gateway.String() - } else { - gatewayIP = "" - } + gatewayIP := "" createOpts := subnets.CreateOpts{ Name: LbProvSubnetName, Description: LbProvSubnetDescription, NetworkID: providerNetwork.ID, TenantID: providerNetwork.TenantID, - CIDR: networkParameters.CIDR.String(), + CIDR: networkParameters.ProviderCIDR.String(), IPVersion: gophercloud.IPVersion(4), AllocationPools: []subnets.AllocationPool{ { - Start: networkParameters.AllocationStart.String(), - End: networkParameters.AllocationEnd.String(), + Start: networkParameters.ProviderAllocationStart.String(), + End: networkParameters.ProviderAllocationEnd.String(), }, }, GatewayIP: &gatewayIP, @@ -346,12 +337,16 @@ func ensureProvNetwork(client *gophercloud.ServiceClient, netDetails *octaviav1. func ensureLbMgmtSubnet( client *gophercloud.ServiceClient, - networkDetails *octaviav1.OctaviaLbMgmtNetworks, tenantNetwork *networks.Network, networkParameters *NetworkParameters, log *logr.Logger, ) (*subnets.Subnet, error) { - ipVersion := networkDetails.SubnetIPVersion + var ipVersion int + if networkParameters.TenantCIDR.Addr().Is6() { + ipVersion = 6 + } else { + ipVersion = 4 + } var createOpts subnets.CreateOpts if ipVersion == 6 { @@ -361,14 +356,14 @@ func ensureLbMgmtSubnet( Description: LbMgmtSubnetDescription, NetworkID: tenantNetwork.ID, TenantID: tenantNetwork.TenantID, - CIDR: LbMgmtSubnetIPv6CIDR, + CIDR: networkParameters.TenantCIDR.String(), IPVersion: gophercloud.IPVersion(ipVersion), IPv6AddressMode: LbMgmtSubnetIPv6AddressMode, IPv6RAMode: LbMgmtSubnetIPv6RAMode, AllocationPools: []subnets.AllocationPool{ { - Start: LbMgmtSubnetIPv6AllocationPoolStart, - End: LbMgmtSubnetIPv6AllocationPoolEnd, + Start: networkParameters.TenantAllocationStart.String(), + End: networkParameters.TenantAllocationEnd.String(), }, }, GatewayIP: &gatewayIP, @@ -381,18 +376,18 @@ func ensureLbMgmtSubnet( Description: LbMgmtSubnetDescription, NetworkID: tenantNetwork.ID, TenantID: tenantNetwork.TenantID, - CIDR: LbMgmtSubnetCIDR, + CIDR: networkParameters.TenantCIDR.String(), IPVersion: gophercloud.IPVersion(ipVersion), AllocationPools: []subnets.AllocationPool{ { - Start: LbMgmtSubnetAllocationPoolStart, - End: LbMgmtSubnetAllocationPoolEnd, + Start: networkParameters.TenantAllocationStart.String(), + End: networkParameters.TenantAllocationEnd.String(), }, }, HostRoutes: []subnets.HostRoute{ { - DestinationCIDR: networkParameters.CIDR.String(), - NextHop: LbMgmtRouterPortIPv4, + DestinationCIDR: networkParameters.ProviderCIDR.String(), + NextHop: networkParameters.TenantGateway.String(), }, }, GatewayIP: &gatewayIP, @@ -435,7 +430,7 @@ func ensureLbMgmtNetwork(client *gophercloud.ServiceClient, networkDetails *octa func externalFixedIPs(subnetID string, networkParameters *NetworkParameters) []routers.ExternalFixedIP { ips := []routers.ExternalFixedIP{ { - IPAddress: networkParameters.RouterIPAddress.String(), + IPAddress: networkParameters.ProviderGateway.String(), SubnetID: subnetID, }, } @@ -784,7 +779,7 @@ func EnsureAmphoraManagementNetwork( if err != nil { return NetworkProvisioningSummary{}, err } - tenantSubnet, err := ensureLbMgmtSubnet(client, netDetails, tenantNetwork, networkParameters, log) + tenantSubnet, err := ensureLbMgmtSubnet(client, tenantNetwork, networkParameters, log) if err != nil { return NetworkProvisioningSummary{}, err } @@ -800,7 +795,7 @@ func EnsureAmphoraManagementNetwork( securityGroups := []string{lbMgmtSecurityGroupID, lbHealthSecurityGroupID} - tenantRouterPort, err := ensurePort(client, tenantNetwork, tenantSubnet, &securityGroups, log) + tenantRouterPort, err := ensurePort(client, tenantNetwork, tenantSubnet, &securityGroups, networkParameters, log) if err != nil { return NetworkProvisioningSummary{}, err } diff --git a/pkg/octavia/network_consts.go b/pkg/octavia/network_consts.go index 73b97e7c..9cf6cfbd 100644 --- a/pkg/octavia/network_consts.go +++ b/pkg/octavia/network_consts.go @@ -35,31 +35,9 @@ const ( // IPv4 consts - // LbMgmtSubnetCIDR - - LbMgmtSubnetCIDR = "172.24.0.0/16" - - // LbMgmtSubnetAllocationPoolStart - - LbMgmtSubnetAllocationPoolStart = "172.24.0.5" - - // LbMgmtSubnetAllocationPoolEnd - - LbMgmtSubnetAllocationPoolEnd = "172.24.255.254" - // LbMgmtSubnetGatewayIP - LbMgmtSubnetGatewayIP = "" - // IPv6 consts - // using Unique local address (fc00::/7) - // with Global ID 6c:6261:6173 ("lbaas") - - // LbMgmtSubnetIPv6CIDR - - LbMgmtSubnetIPv6CIDR = "fd6c:6261:6173:0001::/64" - - // LbMgmtSubnetIPv6AllocationPoolStart - - LbMgmtSubnetIPv6AllocationPoolStart = "fd6c:6261:6173:0001::5" - - // LbMgmtSubnetIPv6AllocationPoolEnd - - LbMgmtSubnetIPv6AllocationPoolEnd = "fd6c:6261:6173:0001:ffff:ffff:ffff:ffff" - // LbMgmtSubnetIPv6AddressMode - LbMgmtSubnetIPv6AddressMode = "slaac" @@ -98,12 +76,6 @@ const ( // LbMgmtRouterPortName LbMgmtRouterPortName = "lb-mgmt-router-port" - // LbMgmtRouterPortIPv4 - LbMgmtRouterPortIPv4 = "172.24.0.3" - - // LbMgmtRouterPortIPv6 - LbMgmtRouterPortIPv6 = "fd6c:6261:6173:0001::3" - // Network attachment details // LbNetworkAttachmentName LbNetworkAttachmentName = "octavia" diff --git a/pkg/octavia/network_parameters.go b/pkg/octavia/network_parameters.go index 292ff31b..6d55e40d 100644 --- a/pkg/octavia/network_parameters.go +++ b/pkg/octavia/network_parameters.go @@ -10,11 +10,14 @@ import ( // NetworkParameters - Parameters for the Octavia networks, based on the config of the NAD type NetworkParameters struct { - CIDR netip.Prefix - AllocationStart netip.Addr - AllocationEnd netip.Addr - Gateway netip.Addr - RouterIPAddress netip.Addr + ProviderCIDR netip.Prefix + ProviderAllocationStart netip.Addr + ProviderAllocationEnd netip.Addr + ProviderGateway netip.Addr + TenantCIDR netip.Prefix + TenantAllocationStart netip.Addr + TenantAllocationEnd netip.Addr + TenantGateway netip.Addr } // NADConfig - IPAM parameters of the NAD @@ -31,7 +34,8 @@ type NADIpam struct { } type NADRoute struct { - Gateway netip.Addr `json:"gw"` + Gateway netip.Addr `json:"gw"` + Destination netip.Prefix `json:"dst"` } func getConfigFromNAD( @@ -47,6 +51,38 @@ func getConfigFromNAD( return nadConfig, nil } +func getRangeAndGatewayFromCIDR( + cidr netip.Prefix, +) (start netip.Addr, end netip.Addr, gateway netip.Addr) { + addr := cidr.Addr() + if addr.Is6() { + addrBytes := addr.As16() + for i := 8; i < 15; i++ { + addrBytes[i] = 0 + } + addrBytes[15] = 3 + gateway = netip.AddrFrom16(addrBytes) + addrBytes[15] = 5 + start = netip.AddrFrom16(addrBytes) + for i := 8; i < 15; i++ { + addrBytes[i] = 0xff + } + addrBytes[15] = 0xfe + end = netip.AddrFrom16(addrBytes) + } else { + addrBytes := addr.As4() + addrBytes[2] = 0 + addrBytes[3] = 3 + gateway = netip.AddrFrom4(addrBytes) + addrBytes[3] = 5 + start = netip.AddrFrom4(addrBytes) + addrBytes[2] = 0xff + addrBytes[3] = 0xfe + end = netip.AddrFrom4(addrBytes) + } + return +} + func GetNetworkParametersFromNAD( nad *networkv1.NetworkAttachmentDefinition, ) (*NetworkParameters, error) { @@ -57,24 +93,41 @@ func GetNetworkParametersFromNAD( return nil, fmt.Errorf("cannot read network parameters: %w", err) } - networkParameters.CIDR = nadConfig.IPAM.CIDR + // Provider subnet parameters + networkParameters.ProviderCIDR = nadConfig.IPAM.CIDR - networkParameters.AllocationStart = nadConfig.IPAM.RangeEnd.Next() - end := networkParameters.AllocationStart + networkParameters.ProviderAllocationStart = nadConfig.IPAM.RangeEnd.Next() + end := networkParameters.ProviderAllocationStart for i := 0; i < LbProvSubnetPoolSize; i++ { - if !networkParameters.CIDR.Contains(end) { - return nil, fmt.Errorf("cannot allocate %d IP addresses in %s", LbProvSubnetPoolSize, networkParameters.CIDR) + if !networkParameters.ProviderCIDR.Contains(end) { + return nil, fmt.Errorf("cannot allocate %d IP addresses in %s", LbProvSubnetPoolSize, networkParameters.ProviderCIDR) } end = end.Next() } - networkParameters.AllocationEnd = end - // TODO(gthiemonge) Remove routes from NAD, manage them in the operator + networkParameters.ProviderAllocationEnd = end if len(nadConfig.IPAM.Routes) > 0 { - networkParameters.RouterIPAddress = nadConfig.IPAM.Routes[0].Gateway + networkParameters.ProviderGateway = nadConfig.IPAM.Routes[0].Gateway } else { return nil, fmt.Errorf("cannot find gateway information in network attachment") } - // Gateway is currently unset + + // Tenant subnet parameters + networkParameters.TenantCIDR = nadConfig.IPAM.Routes[0].Destination + var bitlen int + if networkParameters.TenantCIDR.Addr().Is6() { + bitlen = 64 + } else { + bitlen = 16 + } + + if networkParameters.TenantCIDR.Bits() != bitlen { + return nil, fmt.Errorf("the tenant CIDR is /%d, it should be /%d", networkParameters.TenantCIDR.Bits(), bitlen) + } + + start, end, gateway := getRangeAndGatewayFromCIDR(networkParameters.TenantCIDR) + networkParameters.TenantAllocationStart = start + networkParameters.TenantAllocationEnd = end + networkParameters.TenantGateway = gateway return networkParameters, err } From ec3be4d2d361ab1bf3a7166d7f246a6e72053e3a Mon Sep 17 00:00:00 2001 From: Gregory Thiemonge Date: Fri, 7 Jun 2024 12:01:27 +0000 Subject: [PATCH 3/6] Define routes of the management network in the operator In case the gateway of the management network is not defined in the NAD Select an address in the CIDR and use it for the gateway --- ...enstack.org_octaviaamphoracontrollers.yaml | 6 +++ api/bases/octavia.openstack.org_octavias.yaml | 18 ++++++++ api/v1beta1/amphoracontroller_types.go | 8 ++++ ...enstack.org_octaviaamphoracontrollers.yaml | 6 +++ .../bases/octavia.openstack.org_octavias.yaml | 18 ++++++++ controllers/octavia_controller.go | 2 + pkg/amphoracontrollers/daemonset.go | 3 ++ pkg/octavia/lb_mgmt_network.go | 28 +++++++------ ...r_start.sh => octavia_controller_start.sh} | 8 +++- .../bin/octavia_mgmt_subnet_route.py | 41 +++++++++++++++++++ .../config/octavia-healthmanager-config.json | 2 +- .../config/octavia-housekeeping-config.json | 2 +- .../config/octavia-worker-config.json | 2 +- 13 files changed, 127 insertions(+), 17 deletions(-) rename templates/octaviaamphoracontroller/bin/{octavia_healthmanager_start.sh => octavia_controller_start.sh} (64%) create mode 100755 templates/octaviaamphoracontroller/bin/octavia_mgmt_subnet_route.py diff --git a/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml b/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml index ae97fbb8..5f5fe6eb 100644 --- a/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml +++ b/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml @@ -127,6 +127,12 @@ spec: description: NodeSelector to target subset of worker nodes running this service type: object + octaviaProviderSubnetCIDR: + description: OctaviaProviderSubnetCIDR - + type: string + octaviaProviderSubnetGateway: + description: OctaviaProviderSubnetGateway - + type: string passwordSelectors: default: service: OctaviaPassword diff --git a/api/bases/octavia.openstack.org_octavias.yaml b/api/bases/octavia.openstack.org_octavias.yaml index c83b7364..87cc4c23 100644 --- a/api/bases/octavia.openstack.org_octavias.yaml +++ b/api/bases/octavia.openstack.org_octavias.yaml @@ -580,6 +580,12 @@ spec: description: NodeSelector to target subset of worker nodes running this service type: object + octaviaProviderSubnetCIDR: + description: OctaviaProviderSubnetCIDR - + type: string + octaviaProviderSubnetGateway: + description: OctaviaProviderSubnetGateway - + type: string passwordSelectors: default: service: OctaviaPassword @@ -775,6 +781,12 @@ spec: description: NodeSelector to target subset of worker nodes running this service type: object + octaviaProviderSubnetCIDR: + description: OctaviaProviderSubnetCIDR - + type: string + octaviaProviderSubnetGateway: + description: OctaviaProviderSubnetGateway - + type: string passwordSelectors: default: service: OctaviaPassword @@ -975,6 +987,12 @@ spec: description: NodeSelector to target subset of worker nodes running this service type: object + octaviaProviderSubnetCIDR: + description: OctaviaProviderSubnetCIDR - + type: string + octaviaProviderSubnetGateway: + description: OctaviaProviderSubnetGateway - + type: string passwordSelectors: default: service: OctaviaPassword diff --git a/api/v1beta1/amphoracontroller_types.go b/api/v1beta1/amphoracontroller_types.go index 655eb3e5..c3674a0e 100644 --- a/api/v1beta1/amphoracontroller_types.go +++ b/api/v1beta1/amphoracontroller_types.go @@ -143,6 +143,14 @@ type OctaviaAmphoraControllerSpecCore struct { // +operator-sdk:csv:customresourcedefinitions:type=spec // TLS - Parameters related to the TLS TLS tls.Ca `json:"tls,omitempty"` + + // +kubebuilder:validation:Optional + // OctaviaProviderSubnetGateway - + OctaviaProviderSubnetGateway string `json:"octaviaProviderSubnetGateway"` + + // +kubebuilder:validation:Optional + // OctaviaProviderSubnetCIDR - + OctaviaProviderSubnetCIDR string `json:"octaviaProviderSubnetCIDR"` } // OctaviaAmphoraControllerStatus defines the observed state of the Octavia Amphora Controller diff --git a/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml b/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml index ae97fbb8..5f5fe6eb 100644 --- a/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml +++ b/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml @@ -127,6 +127,12 @@ spec: description: NodeSelector to target subset of worker nodes running this service type: object + octaviaProviderSubnetCIDR: + description: OctaviaProviderSubnetCIDR - + type: string + octaviaProviderSubnetGateway: + description: OctaviaProviderSubnetGateway - + type: string passwordSelectors: default: service: OctaviaPassword diff --git a/config/crd/bases/octavia.openstack.org_octavias.yaml b/config/crd/bases/octavia.openstack.org_octavias.yaml index c83b7364..87cc4c23 100644 --- a/config/crd/bases/octavia.openstack.org_octavias.yaml +++ b/config/crd/bases/octavia.openstack.org_octavias.yaml @@ -580,6 +580,12 @@ spec: description: NodeSelector to target subset of worker nodes running this service type: object + octaviaProviderSubnetCIDR: + description: OctaviaProviderSubnetCIDR - + type: string + octaviaProviderSubnetGateway: + description: OctaviaProviderSubnetGateway - + type: string passwordSelectors: default: service: OctaviaPassword @@ -775,6 +781,12 @@ spec: description: NodeSelector to target subset of worker nodes running this service type: object + octaviaProviderSubnetCIDR: + description: OctaviaProviderSubnetCIDR - + type: string + octaviaProviderSubnetGateway: + description: OctaviaProviderSubnetGateway - + type: string passwordSelectors: default: service: OctaviaPassword @@ -975,6 +987,12 @@ spec: description: NodeSelector to target subset of worker nodes running this service type: object + octaviaProviderSubnetCIDR: + description: OctaviaProviderSubnetCIDR - + type: string + octaviaProviderSubnetGateway: + description: OctaviaProviderSubnetGateway - + type: string passwordSelectors: default: service: OctaviaPassword diff --git a/controllers/octavia_controller.go b/controllers/octavia_controller.go index 74dd5423..75baa807 100644 --- a/controllers/octavia_controller.go +++ b/controllers/octavia_controller.go @@ -1333,6 +1333,8 @@ func (r *OctaviaReconciler) amphoraControllerDaemonSetCreateOrUpdate( daemonset.Spec.AmphoraCustomFlavors = instance.Spec.AmphoraCustomFlavors daemonset.Spec.TLS = instance.Spec.OctaviaAPI.TLS.Ca daemonset.Spec.AmphoraImageOwnerID = ampImageOwnerID + daemonset.Spec.OctaviaProviderSubnetGateway = networkInfo.ManagementSubnetGateway + daemonset.Spec.OctaviaProviderSubnetCIDR = networkInfo.ManagementSubnetCIDR if len(daemonset.Spec.NodeSelector) == 0 { daemonset.Spec.NodeSelector = instance.Spec.NodeSelector } diff --git a/pkg/amphoracontrollers/daemonset.go b/pkg/amphoracontrollers/daemonset.go index 26f29d3f..a5c70faf 100644 --- a/pkg/amphoracontrollers/daemonset.go +++ b/pkg/amphoracontrollers/daemonset.go @@ -84,6 +84,9 @@ func DaemonSet( envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") envVars["CONFIG_HASH"] = env.SetValue(configHash) + envVars["MGMT_CIDR"] = env.SetValue(instance.Spec.OctaviaProviderSubnetCIDR) + envVars["MGMT_GATEWAY"] = env.SetValue(instance.Spec.OctaviaProviderSubnetGateway) + // Add the CA bundle if instance.Spec.TLS.CaBundleSecretName != "" { volumes = append(volumes, instance.Spec.TLS.CreateVolume()) diff --git a/pkg/octavia/lb_mgmt_network.go b/pkg/octavia/lb_mgmt_network.go index 87173688..59efaf5c 100644 --- a/pkg/octavia/lb_mgmt_network.go +++ b/pkg/octavia/lb_mgmt_network.go @@ -33,12 +33,14 @@ import ( ) type NetworkProvisioningSummary struct { - TenantNetworkID string - TenantSubnetID string - TenantRouterPortID string - ProviderNetworkID string - RouterID string - SecurityGroupID string + TenantNetworkID string + TenantSubnetID string + TenantRouterPortID string + ProviderNetworkID string + RouterID string + SecurityGroupID string + ManagementSubnetCIDR string + ManagementSubnetGateway string } // @@ -870,11 +872,13 @@ func EnsureAmphoraManagementNetwork( } return NetworkProvisioningSummary{ - TenantNetworkID: tenantNetwork.ID, - TenantSubnetID: tenantSubnet.ID, - TenantRouterPortID: tenantRouterPort.ID, - ProviderNetworkID: providerNetwork.ID, - RouterID: router.ID, - SecurityGroupID: lbMgmtSecurityGroupID, + TenantNetworkID: tenantNetwork.ID, + TenantSubnetID: tenantSubnet.ID, + TenantRouterPortID: tenantRouterPort.ID, + ProviderNetworkID: providerNetwork.ID, + RouterID: router.ID, + SecurityGroupID: lbMgmtSecurityGroupID, + ManagementSubnetCIDR: networkParameters.TenantCIDR.String(), + ManagementSubnetGateway: networkParameters.ProviderGateway.String(), }, nil } diff --git a/templates/octaviaamphoracontroller/bin/octavia_healthmanager_start.sh b/templates/octaviaamphoracontroller/bin/octavia_controller_start.sh similarity index 64% rename from templates/octaviaamphoracontroller/bin/octavia_healthmanager_start.sh rename to templates/octaviaamphoracontroller/bin/octavia_controller_start.sh index c591dfed..3506006d 100755 --- a/templates/octaviaamphoracontroller/bin/octavia_healthmanager_start.sh +++ b/templates/octaviaamphoracontroller/bin/octavia_controller_start.sh @@ -15,6 +15,10 @@ # under the License. set -ex -/usr/local/bin/container-scripts/octavia_hm_advertisement.py octavia +/usr/local/bin/container-scripts/octavia_mgmt_subnet_route.py octavia "$MGMT_CIDR" "$MGMT_GATEWAY" -exec /usr/bin/octavia-health-manager --config-file /usr/share/octavia/octavia-dist.conf --config-file /etc/octavia/octavia.conf +if [ "$1" = "octavia-health-manager" ]; then + /usr/local/bin/container-scripts/octavia_hm_advertisement.py octavia +fi + +exec /usr/bin/$1 --config-file /usr/share/octavia/octavia-dist.conf --config-file /etc/octavia/octavia.conf diff --git a/templates/octaviaamphoracontroller/bin/octavia_mgmt_subnet_route.py b/templates/octaviaamphoracontroller/bin/octavia_mgmt_subnet_route.py new file mode 100755 index 00000000..f83b5628 --- /dev/null +++ b/templates/octaviaamphoracontroller/bin/octavia_mgmt_subnet_route.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +# +# Copyright 2024 Red Hat Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys +import socket + +from pyroute2 import IPRoute + +try: + interface_name = sys.argv[1] + dst = sys.argv[2] + gateway = sys.argv[3] +except IndexError: + print(f"usage: {sys.argv[0]} ") + sys.exit(1) + +ip = IPRoute() + +try: + idx = ip.link_lookup(ifname=interface_name)[0] +except IndexError: + print(f"Cannot find interface '{interface_name}', skipping") + sys.exit(0) + +try: + ip.route('add', index=idx, dst=dst, gateway=gateway) +except Exception as e: + print(f"Cannot set route {dst} via {gateway}: {e}") diff --git a/templates/octaviaamphoracontroller/config/octavia-healthmanager-config.json b/templates/octaviaamphoracontroller/config/octavia-healthmanager-config.json index 6dd2e42f..c4268fb2 100644 --- a/templates/octaviaamphoracontroller/config/octavia-healthmanager-config.json +++ b/templates/octaviaamphoracontroller/config/octavia-healthmanager-config.json @@ -1,5 +1,5 @@ { - "command": "/usr/local/bin/container-scripts/octavia_healthmanager_start.sh", + "command": "/usr/local/bin/container-scripts/octavia_controller_start.sh octavia-health-manager", "config_files": [ { "source": "/var/lib/config-data/merged/octavia.conf", diff --git a/templates/octaviaamphoracontroller/config/octavia-housekeeping-config.json b/templates/octaviaamphoracontroller/config/octavia-housekeeping-config.json index 5214b675..5e421fed 100644 --- a/templates/octaviaamphoracontroller/config/octavia-housekeeping-config.json +++ b/templates/octaviaamphoracontroller/config/octavia-housekeeping-config.json @@ -1,5 +1,5 @@ { - "command": "/usr/bin/octavia-housekeeping --config-file /usr/share/octavia/octavia-dist.conf --config-file /etc/octavia/octavia.conf", + "command": "/usr/local/bin/container-scripts/octavia_controller_start.sh octavia-housekeeping", "config_files": [ { "source": "/var/lib/config-data/merged/octavia.conf", diff --git a/templates/octaviaamphoracontroller/config/octavia-worker-config.json b/templates/octaviaamphoracontroller/config/octavia-worker-config.json index 4adeaad1..80e33bd6 100644 --- a/templates/octaviaamphoracontroller/config/octavia-worker-config.json +++ b/templates/octaviaamphoracontroller/config/octavia-worker-config.json @@ -1,5 +1,5 @@ { - "command": "/usr/bin/octavia-worker --config-file /usr/share/octavia/octavia-dist.conf --config-file /etc/octavia/octavia.conf", + "command": "/usr/local/bin/container-scripts/octavia_controller_start.sh octavia-worker", "config_files": [ { "source": "/var/lib/config-data/merged/octavia.conf", From 47179b663aaa71cd121057873c52d5308dc9c983 Mon Sep 17 00:00:00 2001 From: Gregory Thiemonge Date: Fri, 7 Jun 2024 15:12:34 +0000 Subject: [PATCH 4/6] Set host_routes after plugging the subnet into the router the previous workflow was - create a subnet with a host route via a fixed ip - create a port with the fixed ip - plug it into the router but in IPv6, fixed ip are denied when automatic addresses are enabled now the code does: - create a subnet - create a port - plug the port into the router - set the ip of the port as a gateway of the subnet --- pkg/octavia/lb_mgmt_network.go | 81 ++++++++++++++++++------------- pkg/octavia/network_parameters.go | 12 ++--- 2 files changed, 51 insertions(+), 42 deletions(-) diff --git a/pkg/octavia/lb_mgmt_network.go b/pkg/octavia/lb_mgmt_network.go index 59efaf5c..17327e86 100644 --- a/pkg/octavia/lb_mgmt_network.go +++ b/pkg/octavia/lb_mgmt_network.go @@ -35,7 +35,6 @@ import ( type NetworkProvisioningSummary struct { TenantNetworkID string TenantSubnetID string - TenantRouterPortID string ProviderNetworkID string RouterID string SecurityGroupID string @@ -48,7 +47,7 @@ type NetworkProvisioningSummary struct { // status. // -func findPort(client *gophercloud.ServiceClient, networkID string, ipAddress string, log *logr.Logger) (*ports.Port, error) { +func findPort(client *gophercloud.ServiceClient, networkID string, name string, log *logr.Logger) (*ports.Port, error) { listOpts := ports.ListOpts{ NetworkID: networkID, } @@ -65,7 +64,7 @@ func findPort(client *gophercloud.ServiceClient, networkID string, ipAddress str } if len(allPorts) > 0 { for _, port := range allPorts { - if len(port.FixedIPs) > 0 && port.FixedIPs[0].IPAddress == ipAddress { + if port.Name == name { return &port, nil } } @@ -73,10 +72,8 @@ func findPort(client *gophercloud.ServiceClient, networkID string, ipAddress str return nil, nil } -func ensurePort(client *gophercloud.ServiceClient, tenantNetwork *networks.Network, tenantSubnet *subnets.Subnet, - securityGroups *[]string, networkParameters *NetworkParameters, log *logr.Logger) (*ports.Port, error) { - ipAddress := networkParameters.TenantGateway.String() - p, err := findPort(client, tenantNetwork.ID, ipAddress, log) +func ensurePort(client *gophercloud.ServiceClient, tenantNetwork *networks.Network, securityGroups *[]string, log *logr.Logger) (*ports.Port, error) { + p, err := findPort(client, tenantNetwork.ID, LbMgmtRouterPortName, log) if err != nil { return nil, err } @@ -89,15 +86,9 @@ func ensurePort(client *gophercloud.ServiceClient, tenantNetwork *networks.Netwo log.Info("Unable to locate port, creating new one") asu := true createOpts := ports.CreateOpts{ - Name: LbMgmtRouterPortName, - AdminStateUp: &asu, - NetworkID: tenantNetwork.ID, - FixedIPs: []ports.IP{ - { - SubnetID: tenantSubnet.ID, - IPAddress: ipAddress, - }, - }, + Name: LbMgmtRouterPortName, + AdminStateUp: &asu, + NetworkID: tenantNetwork.ID, SecurityGroups: securityGroups, } p, err = ports.Create(client, createOpts).Extract() @@ -296,13 +287,19 @@ func ensureProvSubnet( log *logr.Logger, ) (*subnets.Subnet, error) { gatewayIP := "" + var ipVersion int + if networkParameters.ProviderCIDR.Addr().Is6() { + ipVersion = 6 + } else { + ipVersion = 4 + } createOpts := subnets.CreateOpts{ Name: LbProvSubnetName, Description: LbProvSubnetDescription, NetworkID: providerNetwork.ID, TenantID: providerNetwork.TenantID, CIDR: networkParameters.ProviderCIDR.String(), - IPVersion: gophercloud.IPVersion(4), + IPVersion: gophercloud.IPVersion(ipVersion), AllocationPools: []subnets.AllocationPool{ { Start: networkParameters.ProviderAllocationStart.String(), @@ -311,7 +308,7 @@ func ensureProvSubnet( }, GatewayIP: &gatewayIP, } - return ensureSubnet(client, 4, createOpts, log) + return ensureSubnet(client, ipVersion, createOpts, log) } func ensureProvNetwork(client *gophercloud.ServiceClient, netDetails *octaviav1.OctaviaLbMgmtNetworks, serviceTenantID string, log *logr.Logger) ( @@ -337,6 +334,31 @@ func ensureProvNetwork(client *gophercloud.ServiceClient, netDetails *octaviav1. return provNet, nil } +func ensureLbMgmtSubnetRoutes( + client *gophercloud.ServiceClient, + tenantSubnet *subnets.Subnet, + networkParameters *NetworkParameters, + tenantRouterPort *ports.Port, +) error { + if len(tenantSubnet.HostRoutes) == 0 { + hostRoutes := []subnets.HostRoute{ + { + DestinationCIDR: networkParameters.ProviderCIDR.String(), + NextHop: tenantRouterPort.FixedIPs[0].IPAddress, + }, + } + updateOpts := subnets.UpdateOpts{ + HostRoutes: &hostRoutes, + } + _, err := subnets.Update(client, tenantSubnet.ID, updateOpts).Extract() + if err != nil { + return err + } + } + + return nil +} + func ensureLbMgmtSubnet( client *gophercloud.ServiceClient, tenantNetwork *networks.Network, @@ -369,7 +391,6 @@ func ensureLbMgmtSubnet( }, }, GatewayIP: &gatewayIP, - // TODO(beagles): ipv6 host routes } } else { gatewayIP := LbMgmtSubnetGatewayIP @@ -386,12 +407,6 @@ func ensureLbMgmtSubnet( End: networkParameters.TenantAllocationEnd.String(), }, }, - HostRoutes: []subnets.HostRoute{ - { - DestinationCIDR: networkParameters.ProviderCIDR.String(), - NextHop: networkParameters.TenantGateway.String(), - }, - }, GatewayIP: &gatewayIP, } } @@ -797,7 +812,7 @@ func EnsureAmphoraManagementNetwork( securityGroups := []string{lbMgmtSecurityGroupID, lbHealthSecurityGroupID} - tenantRouterPort, err := ensurePort(client, tenantNetwork, tenantSubnet, &securityGroups, networkParameters, log) + tenantRouterPort, err := ensurePort(client, tenantNetwork, &securityGroups, log) if err != nil { return NetworkProvisioningSummary{}, err } @@ -856,8 +871,7 @@ func EnsureAmphoraManagementNetwork( log.Error(err, "Unable to create router object") return NetworkProvisioningSummary{}, err } - } - if tenantRouterPort.DeviceID == "" { + interfaceOpts := routers.AddInterfaceOpts{ PortID: tenantRouterPort.ID, } @@ -865,16 +879,17 @@ func EnsureAmphoraManagementNetwork( if err != nil { log.Error(err, fmt.Sprintf("Unable to add interface port %s to router %s", tenantRouterPort.ID, router.ID)) } - } else if tenantRouterPort.DeviceID != router.ID { - return NetworkProvisioningSummary{}, - fmt.Errorf("Port %s has unexpected device ID %s and cannot be added to router %s", tenantRouterPort.ID, - tenantRouterPort.DeviceID, router.ID) + } + // Set route on subnet + + err = ensureLbMgmtSubnetRoutes(client, tenantSubnet, networkParameters, tenantRouterPort) + if err != nil { + log.Error(err, fmt.Sprintf("Unable to set host routes on subnet %s", tenantSubnet.ID)) } return NetworkProvisioningSummary{ TenantNetworkID: tenantNetwork.ID, TenantSubnetID: tenantSubnet.ID, - TenantRouterPortID: tenantRouterPort.ID, ProviderNetworkID: providerNetwork.ID, RouterID: router.ID, SecurityGroupID: lbMgmtSecurityGroupID, diff --git a/pkg/octavia/network_parameters.go b/pkg/octavia/network_parameters.go index 6d55e40d..393535cd 100644 --- a/pkg/octavia/network_parameters.go +++ b/pkg/octavia/network_parameters.go @@ -17,7 +17,6 @@ type NetworkParameters struct { TenantCIDR netip.Prefix TenantAllocationStart netip.Addr TenantAllocationEnd netip.Addr - TenantGateway netip.Addr } // NADConfig - IPAM parameters of the NAD @@ -51,17 +50,15 @@ func getConfigFromNAD( return nadConfig, nil } -func getRangeAndGatewayFromCIDR( +func getRangeFromCIDR( cidr netip.Prefix, -) (start netip.Addr, end netip.Addr, gateway netip.Addr) { +) (start netip.Addr, end netip.Addr) { addr := cidr.Addr() if addr.Is6() { addrBytes := addr.As16() for i := 8; i < 15; i++ { addrBytes[i] = 0 } - addrBytes[15] = 3 - gateway = netip.AddrFrom16(addrBytes) addrBytes[15] = 5 start = netip.AddrFrom16(addrBytes) for i := 8; i < 15; i++ { @@ -72,8 +69,6 @@ func getRangeAndGatewayFromCIDR( } else { addrBytes := addr.As4() addrBytes[2] = 0 - addrBytes[3] = 3 - gateway = netip.AddrFrom4(addrBytes) addrBytes[3] = 5 start = netip.AddrFrom4(addrBytes) addrBytes[2] = 0xff @@ -124,10 +119,9 @@ func GetNetworkParametersFromNAD( return nil, fmt.Errorf("the tenant CIDR is /%d, it should be /%d", networkParameters.TenantCIDR.Bits(), bitlen) } - start, end, gateway := getRangeAndGatewayFromCIDR(networkParameters.TenantCIDR) + start, end := getRangeFromCIDR(networkParameters.TenantCIDR) networkParameters.TenantAllocationStart = start networkParameters.TenantAllocationEnd = end - networkParameters.TenantGateway = gateway return networkParameters, err } From 01db90270c13b976e8e876fdcdd5c28dc35178d5 Mon Sep 17 00:00:00 2001 From: Gregory Thiemonge Date: Mon, 10 Jun 2024 06:16:39 +0000 Subject: [PATCH 5/6] Add script that displays network config on startup --- .../bin/octavia_controller_start.sh | 3 ++ .../bin/octavia_status.py | 45 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100755 templates/octaviaamphoracontroller/bin/octavia_status.py diff --git a/templates/octaviaamphoracontroller/bin/octavia_controller_start.sh b/templates/octaviaamphoracontroller/bin/octavia_controller_start.sh index 3506006d..8c6792df 100755 --- a/templates/octaviaamphoracontroller/bin/octavia_controller_start.sh +++ b/templates/octaviaamphoracontroller/bin/octavia_controller_start.sh @@ -21,4 +21,7 @@ if [ "$1" = "octavia-health-manager" ]; then /usr/local/bin/container-scripts/octavia_hm_advertisement.py octavia fi +# Ignore possible errors +/usr/local/bin/container-scripts/octavia_status.py || true + exec /usr/bin/$1 --config-file /usr/share/octavia/octavia-dist.conf --config-file /etc/octavia/octavia.conf diff --git a/templates/octaviaamphoracontroller/bin/octavia_status.py b/templates/octaviaamphoracontroller/bin/octavia_status.py new file mode 100755 index 00000000..57f5eff7 --- /dev/null +++ b/templates/octaviaamphoracontroller/bin/octavia_status.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# +# Copyright 2024 Red Hat Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from pyroute2 import IPRoute + +ip = IPRoute() + +ifaces = {} + +for link in ip.get_links(): + attrs = {k: v for k, v in link['attrs']} + ifaces[link['index']] = attrs['IFLA_IFNAME'] + +for addr in ip.get_addr(): + attrs = {k: v for k, v in addr['attrs']} + print(f"addr {attrs['IFA_ADDRESS']}/{addr['prefixlen']} " + f"dev {ifaces[addr['index']]}") + +for route in ip.get_routes(): + attrs = {k: v for k, v in route['attrs']} + if attrs['RTA_TABLE'] != 254: + continue + suffix = f"/{route['dst_len']}" if route['dst_len'] else "" + route_str = f"route {attrs.get('RTA_DST', 'default')}{suffix} " + if attrs.get('RTA_GATEWAY'): + route_str += f"via {attrs.get('RTA_GATEWAY')} " + route_str += f"dev {ifaces[attrs['RTA_OIF']]} " + if attrs.get('RTA_PREFSRC'): + suffix = f"/{route['src_len']}" if route['src_len'] else "" + route_str += f"prefsrc {attrs.get('RTA_PREFSRC')}{suffix} " + + print(route_str) From d6ba3b2544ed03847465629f8643ac781dfb12ce Mon Sep 17 00:00:00 2001 From: Gregory Thiemonge Date: Mon, 10 Jun 2024 13:07:07 +0000 Subject: [PATCH 6/6] Add missing comments for GetNetworkParametersFromNAD --- pkg/octavia/network_parameters.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/pkg/octavia/network_parameters.go b/pkg/octavia/network_parameters.go index 393535cd..27a7cbe7 100644 --- a/pkg/octavia/network_parameters.go +++ b/pkg/octavia/network_parameters.go @@ -50,9 +50,15 @@ func getConfigFromNAD( return nadConfig, nil } +// getRangeFromCIDR - compute a IP address range from a CIDR func getRangeFromCIDR( cidr netip.Prefix, ) (start netip.Addr, end netip.Addr) { + // For IPv6, a /64 is expected, if the CIDR is aaaa:bbbb:cccc:dddd::/64, + // the range is aaaa:bbbb:cccc:dddd::5 - aaaa:bbbb:cccc:dddd:ffff:ffff:ffff:fffe + // For IPv4, a /16 is expected, if the CIDR is a.b.0.0/16 + // the range is a.b.0.5 - a.b.255.254 + // IPs from from 1 to 5 are reserved for later user addr := cidr.Addr() if addr.Is6() { addrBytes := addr.As16() @@ -78,6 +84,7 @@ func getRangeFromCIDR( return } +// GetNetworkParametersFromNAD - Extract network information from the Network Attachment Definition func GetNetworkParametersFromNAD( nad *networkv1.NetworkAttachmentDefinition, ) (*NetworkParameters, error) { @@ -89,8 +96,14 @@ func GetNetworkParametersFromNAD( } // Provider subnet parameters + // These are the parameters for octavia-provider-net/subnet networkParameters.ProviderCIDR = nadConfig.IPAM.CIDR + // OpenShift allocates IP addresses from IPAM.RangeStart to IPAM.RangeEnd + // for the pods. + // We're going to use a range of 25 IP addresses that are assigned to + // the Neutron allocation pool, the range starts right after OpenShift + // RangeEnd. networkParameters.ProviderAllocationStart = nadConfig.IPAM.RangeEnd.Next() end := networkParameters.ProviderAllocationStart for i := 0; i < LbProvSubnetPoolSize; i++ { @@ -100,14 +113,21 @@ func GetNetworkParametersFromNAD( end = end.Next() } networkParameters.ProviderAllocationEnd = end + + // The default gateway of the provider network is the gateway of our route if len(nadConfig.IPAM.Routes) > 0 { networkParameters.ProviderGateway = nadConfig.IPAM.Routes[0].Gateway } else { return nil, fmt.Errorf("cannot find gateway information in network attachment") } - // Tenant subnet parameters + // Tenant subnet parameters - parameters for lb-mgmt-net/subnet + // The NAD must contain one route to the Octavia Tenant Management network, + // the gateway is an IP address of the provider network and the destination + // is the CIDR of the Tenant network. networkParameters.TenantCIDR = nadConfig.IPAM.Routes[0].Destination + + // For IPv4, we require a /16 subnet, for IPv6 a /64 var bitlen int if networkParameters.TenantCIDR.Addr().Is6() { bitlen = 64 @@ -119,6 +139,7 @@ func GetNetworkParametersFromNAD( return nil, fmt.Errorf("the tenant CIDR is /%d, it should be /%d", networkParameters.TenantCIDR.Bits(), bitlen) } + // Compute an allocation range based on the CIDR start, end := getRangeFromCIDR(networkParameters.TenantCIDR) networkParameters.TenantAllocationStart = start networkParameters.TenantAllocationEnd = end