diff --git a/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml b/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml index ae97fbb8..5f5fe6eb 100644 --- a/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml +++ b/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml @@ -127,6 +127,12 @@ spec: description: NodeSelector to target subset of worker nodes running this service type: object + octaviaProviderSubnetCIDR: + description: OctaviaProviderSubnetCIDR - + type: string + octaviaProviderSubnetGateway: + description: OctaviaProviderSubnetGateway - + type: string passwordSelectors: default: service: OctaviaPassword diff --git a/api/bases/octavia.openstack.org_octavias.yaml b/api/bases/octavia.openstack.org_octavias.yaml index 0cce1775..87cc4c23 100644 --- a/api/bases/octavia.openstack.org_octavias.yaml +++ b/api/bases/octavia.openstack.org_octavias.yaml @@ -103,7 +103,6 @@ spec: lbMgmtNetwork: default: manageLbMgmtNetworks: true - subnetIpVersion: 4 description: OctaviaLbMgmtNetworks Settings for Octavia management networks properties: @@ -116,10 +115,6 @@ spec: manageLbMgmtNetworks: default: true type: boolean - subnetIpVersion: - default: 4 - description: IP Version of the managed subnets - type: integer type: object nodeSelector: additionalProperties: @@ -585,6 +580,12 @@ spec: description: NodeSelector to target subset of worker nodes running this service type: object + octaviaProviderSubnetCIDR: + description: OctaviaProviderSubnetCIDR - + type: string + octaviaProviderSubnetGateway: + description: OctaviaProviderSubnetGateway - + type: string passwordSelectors: default: service: OctaviaPassword @@ -780,6 +781,12 @@ spec: description: NodeSelector to target subset of worker nodes running this service type: object + octaviaProviderSubnetCIDR: + description: OctaviaProviderSubnetCIDR - + type: string + octaviaProviderSubnetGateway: + description: OctaviaProviderSubnetGateway - + type: string passwordSelectors: default: service: OctaviaPassword @@ -892,6 +899,11 @@ spec: - secret - serviceAccount type: object + octaviaNetworkAttachment: + default: octavia + description: OctaviaNetworkAttachment is a NetworkAttachment resource + name for the Octavia Management Network + type: string octaviaWorker: description: OctaviaHousekeeping - Spec definition for the Octavia Housekeeping agent for the Octavia deployment @@ -975,6 +987,12 @@ spec: description: NodeSelector to target subset of worker nodes running this service type: object + octaviaProviderSubnetCIDR: + description: OctaviaProviderSubnetCIDR - + type: string + octaviaProviderSubnetGateway: + description: OctaviaProviderSubnetGateway - + type: string passwordSelectors: default: service: OctaviaPassword @@ -1197,6 +1215,7 @@ spec: - apacheContainerImage - databaseInstance - octaviaAPI + - octaviaNetworkAttachment - rabbitMqClusterName - secret type: object diff --git a/api/v1beta1/amphoracontroller_types.go b/api/v1beta1/amphoracontroller_types.go index 655eb3e5..c3674a0e 100644 --- a/api/v1beta1/amphoracontroller_types.go +++ b/api/v1beta1/amphoracontroller_types.go @@ -143,6 +143,14 @@ type OctaviaAmphoraControllerSpecCore struct { // +operator-sdk:csv:customresourcedefinitions:type=spec // TLS - Parameters related to the TLS TLS tls.Ca `json:"tls,omitempty"` + + // +kubebuilder:validation:Optional + // OctaviaProviderSubnetGateway - + OctaviaProviderSubnetGateway string `json:"octaviaProviderSubnetGateway"` + + // +kubebuilder:validation:Optional + // OctaviaProviderSubnetCIDR - + OctaviaProviderSubnetCIDR string `json:"octaviaProviderSubnetCIDR"` } // OctaviaAmphoraControllerStatus defines the observed state of the Octavia Amphora Controller diff --git a/api/v1beta1/octavia_types.go b/api/v1beta1/octavia_types.go index ff6dc37c..10bf7980 100644 --- a/api/v1beta1/octavia_types.go +++ b/api/v1beta1/octavia_types.go @@ -160,7 +160,7 @@ type OctaviaSpecBase struct { TenantName string `json:"tenantName"` // +kubebuilder:validation:Optional - // +kubebuilder:default={manageLbMgmtNetworks: true, subnetIpVersion: 4} + // +kubebuilder:default={manageLbMgmtNetworks: true} LbMgmtNetworks OctaviaLbMgmtNetworks `json:"lbMgmtNetwork"` // +kubebuilder:validation:Optional @@ -192,6 +192,11 @@ type OctaviaSpecBase struct { // +kubebuilder:validation:Required // Apache Container Image URL ApacheContainerImage string `json:"apacheContainerImage"` + + // +kubebuilder:validation:Required + // +kubebuilder:default=octavia + // OctaviaNetworkAttachment is a NetworkAttachment resource name for the Octavia Management Network + OctaviaNetworkAttachment string `json:"octaviaNetworkAttachment"` } // PasswordSelector to identify the DB and AdminUser password from the Secret @@ -213,11 +218,6 @@ type OctaviaLbMgmtNetworks struct { // +kubebuilder:default=true ManageLbMgmtNetworks bool `json:"manageLbMgmtNetworks,omitempty"` - // +kubebuilder:validation:Optional - // +kubebuilder:default=4 - // IP Version of the managed subnets - SubnetIPVersion int `json:"subnetIpVersion,omitempty"` - // +kubebuilder:validation:Optional // Availability zones for the octavia management network resources AvailabilityZones []string `json:"availabilityZones,omitempty"` diff --git a/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml b/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml index ae97fbb8..5f5fe6eb 100644 --- a/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml +++ b/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml @@ -127,6 +127,12 @@ spec: description: NodeSelector to target subset of worker nodes running this service type: object + octaviaProviderSubnetCIDR: + description: OctaviaProviderSubnetCIDR - + type: string + octaviaProviderSubnetGateway: + description: OctaviaProviderSubnetGateway - + type: string passwordSelectors: default: service: OctaviaPassword diff --git a/config/crd/bases/octavia.openstack.org_octavias.yaml b/config/crd/bases/octavia.openstack.org_octavias.yaml index 0cce1775..87cc4c23 100644 --- a/config/crd/bases/octavia.openstack.org_octavias.yaml +++ b/config/crd/bases/octavia.openstack.org_octavias.yaml @@ -103,7 +103,6 @@ spec: lbMgmtNetwork: default: manageLbMgmtNetworks: true - subnetIpVersion: 4 description: OctaviaLbMgmtNetworks Settings for Octavia management networks properties: @@ -116,10 +115,6 @@ spec: manageLbMgmtNetworks: default: true type: boolean - subnetIpVersion: - default: 4 - description: IP Version of the managed subnets - type: integer type: object nodeSelector: additionalProperties: @@ -585,6 +580,12 @@ spec: description: NodeSelector to target subset of worker nodes running this service type: object + octaviaProviderSubnetCIDR: + description: OctaviaProviderSubnetCIDR - + type: string + octaviaProviderSubnetGateway: + description: OctaviaProviderSubnetGateway - + type: string passwordSelectors: default: service: OctaviaPassword @@ -780,6 +781,12 @@ spec: description: NodeSelector to target subset of worker nodes running this service type: object + octaviaProviderSubnetCIDR: + description: OctaviaProviderSubnetCIDR - + type: string + octaviaProviderSubnetGateway: + description: OctaviaProviderSubnetGateway - + type: string passwordSelectors: default: service: OctaviaPassword @@ -892,6 +899,11 @@ spec: - secret - serviceAccount type: object + octaviaNetworkAttachment: + default: octavia + description: OctaviaNetworkAttachment is a NetworkAttachment resource + name for the Octavia Management Network + type: string octaviaWorker: description: OctaviaHousekeeping - Spec definition for the Octavia Housekeeping agent for the Octavia deployment @@ -975,6 +987,12 @@ spec: description: NodeSelector to target subset of worker nodes running this service type: object + octaviaProviderSubnetCIDR: + description: OctaviaProviderSubnetCIDR - + type: string + octaviaProviderSubnetGateway: + description: OctaviaProviderSubnetGateway - + type: string passwordSelectors: default: service: OctaviaPassword @@ -1197,6 +1215,7 @@ spec: - apacheContainerImage - databaseInstance - octaviaAPI + - octaviaNetworkAttachment - rabbitMqClusterName - secret type: object diff --git a/controllers/octavia_controller.go b/controllers/octavia_controller.go index e780c740..75baa807 100644 --- a/controllers/octavia_controller.go +++ b/controllers/octavia_controller.go @@ -642,6 +642,16 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav // Amphora reconciliation // ------------------------------------------------------------------------------------------------------------ + nad, err := nad.GetNADWithName(ctx, helper, instance.Spec.OctaviaNetworkAttachment, instance.Namespace) + if err != nil { + return ctrl.Result{}, err + } + + networkParameters, err := octavia.GetNetworkParametersFromNAD(nad) + if err != nil { + return ctrl.Result{}, err + } + // Create load balancer management network and get its Id (networkInfo is actually a struct and contains // multiple details. networkInfo, err := octavia.EnsureAmphoraManagementNetwork( @@ -649,6 +659,7 @@ func (r *OctaviaReconciler) reconcileNormal(ctx context.Context, instance *octav instance.Namespace, instance.Spec.TenantName, &instance.Spec.LbMgmtNetworks, + networkParameters, &Log, helper, ) @@ -1322,6 +1333,8 @@ func (r *OctaviaReconciler) amphoraControllerDaemonSetCreateOrUpdate( daemonset.Spec.AmphoraCustomFlavors = instance.Spec.AmphoraCustomFlavors daemonset.Spec.TLS = instance.Spec.OctaviaAPI.TLS.Ca daemonset.Spec.AmphoraImageOwnerID = ampImageOwnerID + daemonset.Spec.OctaviaProviderSubnetGateway = networkInfo.ManagementSubnetGateway + daemonset.Spec.OctaviaProviderSubnetCIDR = networkInfo.ManagementSubnetCIDR if len(daemonset.Spec.NodeSelector) == 0 { daemonset.Spec.NodeSelector = instance.Spec.NodeSelector } diff --git a/pkg/amphoracontrollers/daemonset.go b/pkg/amphoracontrollers/daemonset.go index 26f29d3f..a5c70faf 100644 --- a/pkg/amphoracontrollers/daemonset.go +++ b/pkg/amphoracontrollers/daemonset.go @@ -84,6 +84,9 @@ func DaemonSet( envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") envVars["CONFIG_HASH"] = env.SetValue(configHash) + envVars["MGMT_CIDR"] = env.SetValue(instance.Spec.OctaviaProviderSubnetCIDR) + envVars["MGMT_GATEWAY"] = env.SetValue(instance.Spec.OctaviaProviderSubnetGateway) + // Add the CA bundle if instance.Spec.TLS.CaBundleSecretName != "" { volumes = append(volumes, instance.Spec.TLS.CreateVolume()) diff --git a/pkg/octavia/lb_mgmt_network.go b/pkg/octavia/lb_mgmt_network.go index 1e2fb698..17327e86 100644 --- a/pkg/octavia/lb_mgmt_network.go +++ b/pkg/octavia/lb_mgmt_network.go @@ -33,12 +33,13 @@ import ( ) type NetworkProvisioningSummary struct { - TenantNetworkID string - TenantSubnetID string - TenantRouterPortID string - ProviderNetworkID string - RouterID string - SecurityGroupID string + TenantNetworkID string + TenantSubnetID string + ProviderNetworkID string + RouterID string + SecurityGroupID string + ManagementSubnetCIDR string + ManagementSubnetGateway string } // @@ -46,7 +47,7 @@ type NetworkProvisioningSummary struct { // status. // -func findPort(client *gophercloud.ServiceClient, networkID string, ipAddress string, log *logr.Logger) (*ports.Port, error) { +func findPort(client *gophercloud.ServiceClient, networkID string, name string, log *logr.Logger) (*ports.Port, error) { listOpts := ports.ListOpts{ NetworkID: networkID, } @@ -63,7 +64,7 @@ func findPort(client *gophercloud.ServiceClient, networkID string, ipAddress str } if len(allPorts) > 0 { for _, port := range allPorts { - if len(port.FixedIPs) > 0 && port.FixedIPs[0].IPAddress == ipAddress { + if port.Name == name { return &port, nil } } @@ -71,14 +72,8 @@ func findPort(client *gophercloud.ServiceClient, networkID string, ipAddress str return nil, nil } -func ensurePort(client *gophercloud.ServiceClient, tenantNetwork *networks.Network, tenantSubnet *subnets.Subnet, - securityGroups *[]string, log *logr.Logger) (*ports.Port, error) { - ipAddress := LbMgmtRouterPortIPv4 - if tenantSubnet.IPVersion == 6 { - ipAddress = LbMgmtRouterPortIPv6 - } - - p, err := findPort(client, tenantNetwork.ID, ipAddress, log) +func ensurePort(client *gophercloud.ServiceClient, tenantNetwork *networks.Network, securityGroups *[]string, log *logr.Logger) (*ports.Port, error) { + p, err := findPort(client, tenantNetwork.ID, LbMgmtRouterPortName, log) if err != nil { return nil, err } @@ -91,15 +86,9 @@ func ensurePort(client *gophercloud.ServiceClient, tenantNetwork *networks.Netwo log.Info("Unable to locate port, creating new one") asu := true createOpts := ports.CreateOpts{ - Name: LbMgmtRouterPortName, - AdminStateUp: &asu, - NetworkID: tenantNetwork.ID, - FixedIPs: []ports.IP{ - { - SubnetID: tenantSubnet.ID, - IPAddress: ipAddress, - }, - }, + Name: LbMgmtRouterPortName, + AdminStateUp: &asu, + NetworkID: tenantNetwork.ID, SecurityGroups: securityGroups, } p, err = ports.Create(client, createOpts).Extract() @@ -291,25 +280,35 @@ func ensureNetworkExt(client *gophercloud.ServiceClient, createOpts networks.Cre return foundNetwork, nil } -func ensureProvSubnet(client *gophercloud.ServiceClient, providerNetwork *networks.Network, log *logr.Logger) ( - *subnets.Subnet, error) { - gatewayIP := LbProvSubnetGatewayIP +func ensureProvSubnet( + client *gophercloud.ServiceClient, + providerNetwork *networks.Network, + networkParameters *NetworkParameters, + log *logr.Logger, +) (*subnets.Subnet, error) { + gatewayIP := "" + var ipVersion int + if networkParameters.ProviderCIDR.Addr().Is6() { + ipVersion = 6 + } else { + ipVersion = 4 + } createOpts := subnets.CreateOpts{ Name: LbProvSubnetName, Description: LbProvSubnetDescription, NetworkID: providerNetwork.ID, TenantID: providerNetwork.TenantID, - CIDR: LbProvSubnetCIDR, - IPVersion: gophercloud.IPVersion(4), + CIDR: networkParameters.ProviderCIDR.String(), + IPVersion: gophercloud.IPVersion(ipVersion), AllocationPools: []subnets.AllocationPool{ { - Start: LbProvSubnetAllocationPoolStart, - End: LbProvSubnetAllocationPoolEnd, + Start: networkParameters.ProviderAllocationStart.String(), + End: networkParameters.ProviderAllocationEnd.String(), }, }, GatewayIP: &gatewayIP, } - return ensureSubnet(client, 4, createOpts, log) + return ensureSubnet(client, ipVersion, createOpts, log) } func ensureProvNetwork(client *gophercloud.ServiceClient, netDetails *octaviav1.OctaviaLbMgmtNetworks, serviceTenantID string, log *logr.Logger) ( @@ -335,13 +334,43 @@ func ensureProvNetwork(client *gophercloud.ServiceClient, netDetails *octaviav1. return provNet, nil } +func ensureLbMgmtSubnetRoutes( + client *gophercloud.ServiceClient, + tenantSubnet *subnets.Subnet, + networkParameters *NetworkParameters, + tenantRouterPort *ports.Port, +) error { + if len(tenantSubnet.HostRoutes) == 0 { + hostRoutes := []subnets.HostRoute{ + { + DestinationCIDR: networkParameters.ProviderCIDR.String(), + NextHop: tenantRouterPort.FixedIPs[0].IPAddress, + }, + } + updateOpts := subnets.UpdateOpts{ + HostRoutes: &hostRoutes, + } + _, err := subnets.Update(client, tenantSubnet.ID, updateOpts).Extract() + if err != nil { + return err + } + } + + return nil +} + func ensureLbMgmtSubnet( client *gophercloud.ServiceClient, - networkDetails *octaviav1.OctaviaLbMgmtNetworks, tenantNetwork *networks.Network, + networkParameters *NetworkParameters, log *logr.Logger, ) (*subnets.Subnet, error) { - ipVersion := networkDetails.SubnetIPVersion + var ipVersion int + if networkParameters.TenantCIDR.Addr().Is6() { + ipVersion = 6 + } else { + ipVersion = 4 + } var createOpts subnets.CreateOpts if ipVersion == 6 { @@ -351,18 +380,17 @@ func ensureLbMgmtSubnet( Description: LbMgmtSubnetDescription, NetworkID: tenantNetwork.ID, TenantID: tenantNetwork.TenantID, - CIDR: LbMgmtSubnetIPv6CIDR, + CIDR: networkParameters.TenantCIDR.String(), IPVersion: gophercloud.IPVersion(ipVersion), IPv6AddressMode: LbMgmtSubnetIPv6AddressMode, IPv6RAMode: LbMgmtSubnetIPv6RAMode, AllocationPools: []subnets.AllocationPool{ { - Start: LbMgmtSubnetIPv6AllocationPoolStart, - End: LbMgmtSubnetIPv6AllocationPoolEnd, + Start: networkParameters.TenantAllocationStart.String(), + End: networkParameters.TenantAllocationEnd.String(), }, }, GatewayIP: &gatewayIP, - // TODO(beagles): ipv6 host routes } } else { gatewayIP := LbMgmtSubnetGatewayIP @@ -371,18 +399,12 @@ func ensureLbMgmtSubnet( Description: LbMgmtSubnetDescription, NetworkID: tenantNetwork.ID, TenantID: tenantNetwork.TenantID, - CIDR: LbMgmtSubnetCIDR, + CIDR: networkParameters.TenantCIDR.String(), IPVersion: gophercloud.IPVersion(ipVersion), AllocationPools: []subnets.AllocationPool{ { - Start: LbMgmtSubnetAllocationPoolStart, - End: LbMgmtSubnetAllocationPoolEnd, - }, - }, - HostRoutes: []subnets.HostRoute{ - { - DestinationCIDR: LbProvSubnetCIDR, - NextHop: LbMgmtRouterPortIPv4, + Start: networkParameters.TenantAllocationStart.String(), + End: networkParameters.TenantAllocationEnd.String(), }, }, GatewayIP: &gatewayIP, @@ -422,10 +444,10 @@ func ensureLbMgmtNetwork(client *gophercloud.ServiceClient, networkDetails *octa return mgmtNetwork, nil } -func externalFixedIPs(subnetID string) []routers.ExternalFixedIP { +func externalFixedIPs(subnetID string, networkParameters *NetworkParameters) []routers.ExternalFixedIP { ips := []routers.ExternalFixedIP{ { - IPAddress: LbRouterFixedIPAddress, + IPAddress: networkParameters.ProviderGateway.String(), SubnetID: subnetID, }, } @@ -452,6 +474,7 @@ func compareExternalFixedIPs(a []routers.ExternalFixedIP, b []routers.ExternalFi func reconcileRouter(client *gophercloud.ServiceClient, router *routers.Router, gatewayNetwork *networks.Network, gatewaySubnet *subnets.Subnet, + networkParameters *NetworkParameters, log *logr.Logger) (*routers.Router, error) { if !router.AdminStateUp { @@ -464,7 +487,7 @@ func reconcileRouter(client *gophercloud.ServiceClient, router *routers.Router, needsUpdate := false updateInfo := routers.UpdateOpts{} enableSNAT := false - fixedIPs := externalFixedIPs(gatewaySubnet.ID) + fixedIPs := externalFixedIPs(gatewaySubnet.ID, networkParameters) // // TODO(beagles) we don't care about the other fields right now because we @@ -752,6 +775,7 @@ func EnsureAmphoraManagementNetwork( ns string, tenantName string, netDetails *octaviav1.OctaviaLbMgmtNetworks, + networkParameters *NetworkParameters, log *logr.Logger, helper *helper.Helper, ) (NetworkProvisioningSummary, error) { @@ -772,7 +796,7 @@ func EnsureAmphoraManagementNetwork( if err != nil { return NetworkProvisioningSummary{}, err } - tenantSubnet, err := ensureLbMgmtSubnet(client, netDetails, tenantNetwork, log) + tenantSubnet, err := ensureLbMgmtSubnet(client, tenantNetwork, networkParameters, log) if err != nil { return NetworkProvisioningSummary{}, err } @@ -788,7 +812,7 @@ func EnsureAmphoraManagementNetwork( securityGroups := []string{lbMgmtSecurityGroupID, lbHealthSecurityGroupID} - tenantRouterPort, err := ensurePort(client, tenantNetwork, tenantSubnet, &securityGroups, log) + tenantRouterPort, err := ensurePort(client, tenantNetwork, &securityGroups, log) if err != nil { return NetworkProvisioningSummary{}, err } @@ -811,7 +835,7 @@ func EnsureAmphoraManagementNetwork( return NetworkProvisioningSummary{}, err } - providerSubnet, err := ensureProvSubnet(client, providerNetwork, log) + providerSubnet, err := ensureProvSubnet(client, providerNetwork, networkParameters, log) if err != nil { return NetworkProvisioningSummary{}, err } @@ -822,7 +846,7 @@ func EnsureAmphoraManagementNetwork( } if router != nil { log.Info("Router object found, reconciling") - router, err = reconcileRouter(client, router, providerNetwork, providerSubnet, log) + router, err = reconcileRouter(client, router, providerNetwork, providerSubnet, networkParameters, log) if err != nil { return NetworkProvisioningSummary{}, err } @@ -833,7 +857,7 @@ func EnsureAmphoraManagementNetwork( gatewayInfo := routers.GatewayInfo{ NetworkID: providerNetwork.ID, EnableSNAT: &enableSNAT, - ExternalFixedIPs: externalFixedIPs(providerSubnet.ID), + ExternalFixedIPs: externalFixedIPs(providerSubnet.ID, networkParameters), } adminStateUp := true createOpts := routers.CreateOpts{ @@ -847,8 +871,7 @@ func EnsureAmphoraManagementNetwork( log.Error(err, "Unable to create router object") return NetworkProvisioningSummary{}, err } - } - if tenantRouterPort.DeviceID == "" { + interfaceOpts := routers.AddInterfaceOpts{ PortID: tenantRouterPort.ID, } @@ -856,18 +879,21 @@ func EnsureAmphoraManagementNetwork( if err != nil { log.Error(err, fmt.Sprintf("Unable to add interface port %s to router %s", tenantRouterPort.ID, router.ID)) } - } else if tenantRouterPort.DeviceID != router.ID { - return NetworkProvisioningSummary{}, - fmt.Errorf("Port %s has unexpected device ID %s and cannot be added to router %s", tenantRouterPort.ID, - tenantRouterPort.DeviceID, router.ID) + } + // Set route on subnet + + err = ensureLbMgmtSubnetRoutes(client, tenantSubnet, networkParameters, tenantRouterPort) + if err != nil { + log.Error(err, fmt.Sprintf("Unable to set host routes on subnet %s", tenantSubnet.ID)) } return NetworkProvisioningSummary{ - TenantNetworkID: tenantNetwork.ID, - TenantSubnetID: tenantSubnet.ID, - TenantRouterPortID: tenantRouterPort.ID, - ProviderNetworkID: providerNetwork.ID, - RouterID: router.ID, - SecurityGroupID: lbMgmtSecurityGroupID, + TenantNetworkID: tenantNetwork.ID, + TenantSubnetID: tenantSubnet.ID, + ProviderNetworkID: providerNetwork.ID, + RouterID: router.ID, + SecurityGroupID: lbMgmtSecurityGroupID, + ManagementSubnetCIDR: networkParameters.TenantCIDR.String(), + ManagementSubnetGateway: networkParameters.ProviderGateway.String(), }, nil } diff --git a/pkg/octavia/network_consts.go b/pkg/octavia/network_consts.go index d2c3c55e..9cf6cfbd 100644 --- a/pkg/octavia/network_consts.go +++ b/pkg/octavia/network_consts.go @@ -35,31 +35,9 @@ const ( // IPv4 consts - // LbMgmtSubnetCIDR - - LbMgmtSubnetCIDR = "172.24.0.0/16" - - // LbMgmtSubnetAllocationPoolStart - - LbMgmtSubnetAllocationPoolStart = "172.24.0.5" - - // LbMgmtSubnetAllocationPoolEnd - - LbMgmtSubnetAllocationPoolEnd = "172.24.255.254" - // LbMgmtSubnetGatewayIP - LbMgmtSubnetGatewayIP = "" - // IPv6 consts - // using Unique local address (fc00::/7) - // with Global ID 6c:6261:6173 ("lbaas") - - // LbMgmtSubnetIPv6CIDR - - LbMgmtSubnetIPv6CIDR = "fd6c:6261:6173:0001::/64" - - // LbMgmtSubnetIPv6AllocationPoolStart - - LbMgmtSubnetIPv6AllocationPoolStart = "fd6c:6261:6173:0001::5" - - // LbMgmtSubnetIPv6AllocationPoolEnd - - LbMgmtSubnetIPv6AllocationPoolEnd = "fd6c:6261:6173:0001:ffff:ffff:ffff:ffff" - // LbMgmtSubnetIPv6AddressMode - LbMgmtSubnetIPv6AddressMode = "slaac" @@ -83,19 +61,10 @@ const ( // LbProvSubnetDescription - LbProvSubnetDescription = "LBaaS Management Provider Subnet" - // IPv4 consts + // LbProvSubnetPoolSize - + LbProvSubnetPoolSize = 25 - // LbProvSubnetCIDR - - LbProvSubnetCIDR = "172.23.0.0/24" - - // LbProvSubnetAllocationPoolStart - - LbProvSubnetAllocationPoolStart = "172.23.0.100" - - // LbProvSubnetAllocationPoolEnd - - LbProvSubnetAllocationPoolEnd = "172.23.0.125" - - // LbProvSubnetGatewayIP - - LbProvSubnetGatewayIP = "" + // IPv4 consts // TODO(beagles): support IPv6 for the provider network. // LbRouterName - @@ -104,18 +73,9 @@ const ( // LbProvPhysicalNet - LbProvPhysicalNet = "octavia" - // LbRouterFixedIPAddress - LbRouterFixedIPAddress = "172.23.0.150" - // LbMgmtRouterPortName LbMgmtRouterPortName = "lb-mgmt-router-port" - // LbMgmtRouterPortIPv4 - LbMgmtRouterPortIPv4 = "172.24.0.3" - - // LbMgmtRouterPortIPv6 - LbMgmtRouterPortIPv6 = "fd6c:6261:6173:0001::3" - // Network attachment details // LbNetworkAttachmentName LbNetworkAttachmentName = "octavia" diff --git a/pkg/octavia/network_parameters.go b/pkg/octavia/network_parameters.go new file mode 100644 index 00000000..27a7cbe7 --- /dev/null +++ b/pkg/octavia/network_parameters.go @@ -0,0 +1,148 @@ +package octavia + +import ( + "encoding/json" + "fmt" + "net/netip" + + networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" +) + +// NetworkParameters - Parameters for the Octavia networks, based on the config of the NAD +type NetworkParameters struct { + ProviderCIDR netip.Prefix + ProviderAllocationStart netip.Addr + ProviderAllocationEnd netip.Addr + ProviderGateway netip.Addr + TenantCIDR netip.Prefix + TenantAllocationStart netip.Addr + TenantAllocationEnd netip.Addr +} + +// NADConfig - IPAM parameters of the NAD +type NADConfig struct { + IPAM NADIpam `json:"ipam"` +} + +type NADIpam struct { + CIDR netip.Prefix `json:"range"` + RangeStart netip.Addr `json:"range_start"` + RangeEnd netip.Addr `json:"range_end"` + + Routes []NADRoute `json:"routes"` +} + +type NADRoute struct { + Gateway netip.Addr `json:"gw"` + Destination netip.Prefix `json:"dst"` +} + +func getConfigFromNAD( + nad *networkv1.NetworkAttachmentDefinition, +) (*NADConfig, error) { + nadConfig := &NADConfig{} + jsonDoc := []byte(nad.Spec.Config) + err := json.Unmarshal(jsonDoc, nadConfig) + if err != nil { + return nil, err + } + + return nadConfig, nil +} + +// getRangeFromCIDR - compute a IP address range from a CIDR +func getRangeFromCIDR( + cidr netip.Prefix, +) (start netip.Addr, end netip.Addr) { + // For IPv6, a /64 is expected, if the CIDR is aaaa:bbbb:cccc:dddd::/64, + // the range is aaaa:bbbb:cccc:dddd::5 - aaaa:bbbb:cccc:dddd:ffff:ffff:ffff:fffe + // For IPv4, a /16 is expected, if the CIDR is a.b.0.0/16 + // the range is a.b.0.5 - a.b.255.254 + // IPs from from 1 to 5 are reserved for later user + addr := cidr.Addr() + if addr.Is6() { + addrBytes := addr.As16() + for i := 8; i < 15; i++ { + addrBytes[i] = 0 + } + addrBytes[15] = 5 + start = netip.AddrFrom16(addrBytes) + for i := 8; i < 15; i++ { + addrBytes[i] = 0xff + } + addrBytes[15] = 0xfe + end = netip.AddrFrom16(addrBytes) + } else { + addrBytes := addr.As4() + addrBytes[2] = 0 + addrBytes[3] = 5 + start = netip.AddrFrom4(addrBytes) + addrBytes[2] = 0xff + addrBytes[3] = 0xfe + end = netip.AddrFrom4(addrBytes) + } + return +} + +// GetNetworkParametersFromNAD - Extract network information from the Network Attachment Definition +func GetNetworkParametersFromNAD( + nad *networkv1.NetworkAttachmentDefinition, +) (*NetworkParameters, error) { + networkParameters := &NetworkParameters{} + + nadConfig, err := getConfigFromNAD(nad) + if err != nil { + return nil, fmt.Errorf("cannot read network parameters: %w", err) + } + + // Provider subnet parameters + // These are the parameters for octavia-provider-net/subnet + networkParameters.ProviderCIDR = nadConfig.IPAM.CIDR + + // OpenShift allocates IP addresses from IPAM.RangeStart to IPAM.RangeEnd + // for the pods. + // We're going to use a range of 25 IP addresses that are assigned to + // the Neutron allocation pool, the range starts right after OpenShift + // RangeEnd. + networkParameters.ProviderAllocationStart = nadConfig.IPAM.RangeEnd.Next() + end := networkParameters.ProviderAllocationStart + for i := 0; i < LbProvSubnetPoolSize; i++ { + if !networkParameters.ProviderCIDR.Contains(end) { + return nil, fmt.Errorf("cannot allocate %d IP addresses in %s", LbProvSubnetPoolSize, networkParameters.ProviderCIDR) + } + end = end.Next() + } + networkParameters.ProviderAllocationEnd = end + + // The default gateway of the provider network is the gateway of our route + if len(nadConfig.IPAM.Routes) > 0 { + networkParameters.ProviderGateway = nadConfig.IPAM.Routes[0].Gateway + } else { + return nil, fmt.Errorf("cannot find gateway information in network attachment") + } + + // Tenant subnet parameters - parameters for lb-mgmt-net/subnet + // The NAD must contain one route to the Octavia Tenant Management network, + // the gateway is an IP address of the provider network and the destination + // is the CIDR of the Tenant network. + networkParameters.TenantCIDR = nadConfig.IPAM.Routes[0].Destination + + // For IPv4, we require a /16 subnet, for IPv6 a /64 + var bitlen int + if networkParameters.TenantCIDR.Addr().Is6() { + bitlen = 64 + } else { + bitlen = 16 + } + + if networkParameters.TenantCIDR.Bits() != bitlen { + return nil, fmt.Errorf("the tenant CIDR is /%d, it should be /%d", networkParameters.TenantCIDR.Bits(), bitlen) + } + + // Compute an allocation range based on the CIDR + start, end := getRangeFromCIDR(networkParameters.TenantCIDR) + networkParameters.TenantAllocationStart = start + networkParameters.TenantAllocationEnd = end + + return networkParameters, err +} diff --git a/templates/octaviaamphoracontroller/bin/octavia_healthmanager_start.sh b/templates/octaviaamphoracontroller/bin/octavia_controller_start.sh similarity index 59% rename from templates/octaviaamphoracontroller/bin/octavia_healthmanager_start.sh rename to templates/octaviaamphoracontroller/bin/octavia_controller_start.sh index c591dfed..8c6792df 100755 --- a/templates/octaviaamphoracontroller/bin/octavia_healthmanager_start.sh +++ b/templates/octaviaamphoracontroller/bin/octavia_controller_start.sh @@ -15,6 +15,13 @@ # under the License. set -ex -/usr/local/bin/container-scripts/octavia_hm_advertisement.py octavia +/usr/local/bin/container-scripts/octavia_mgmt_subnet_route.py octavia "$MGMT_CIDR" "$MGMT_GATEWAY" -exec /usr/bin/octavia-health-manager --config-file /usr/share/octavia/octavia-dist.conf --config-file /etc/octavia/octavia.conf +if [ "$1" = "octavia-health-manager" ]; then + /usr/local/bin/container-scripts/octavia_hm_advertisement.py octavia +fi + +# Ignore possible errors +/usr/local/bin/container-scripts/octavia_status.py || true + +exec /usr/bin/$1 --config-file /usr/share/octavia/octavia-dist.conf --config-file /etc/octavia/octavia.conf diff --git a/templates/octaviaamphoracontroller/bin/octavia_mgmt_subnet_route.py b/templates/octaviaamphoracontroller/bin/octavia_mgmt_subnet_route.py new file mode 100755 index 00000000..f83b5628 --- /dev/null +++ b/templates/octaviaamphoracontroller/bin/octavia_mgmt_subnet_route.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +# +# Copyright 2024 Red Hat Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys +import socket + +from pyroute2 import IPRoute + +try: + interface_name = sys.argv[1] + dst = sys.argv[2] + gateway = sys.argv[3] +except IndexError: + print(f"usage: {sys.argv[0]} ") + sys.exit(1) + +ip = IPRoute() + +try: + idx = ip.link_lookup(ifname=interface_name)[0] +except IndexError: + print(f"Cannot find interface '{interface_name}', skipping") + sys.exit(0) + +try: + ip.route('add', index=idx, dst=dst, gateway=gateway) +except Exception as e: + print(f"Cannot set route {dst} via {gateway}: {e}") diff --git a/templates/octaviaamphoracontroller/bin/octavia_status.py b/templates/octaviaamphoracontroller/bin/octavia_status.py new file mode 100755 index 00000000..57f5eff7 --- /dev/null +++ b/templates/octaviaamphoracontroller/bin/octavia_status.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# +# Copyright 2024 Red Hat Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from pyroute2 import IPRoute + +ip = IPRoute() + +ifaces = {} + +for link in ip.get_links(): + attrs = {k: v for k, v in link['attrs']} + ifaces[link['index']] = attrs['IFLA_IFNAME'] + +for addr in ip.get_addr(): + attrs = {k: v for k, v in addr['attrs']} + print(f"addr {attrs['IFA_ADDRESS']}/{addr['prefixlen']} " + f"dev {ifaces[addr['index']]}") + +for route in ip.get_routes(): + attrs = {k: v for k, v in route['attrs']} + if attrs['RTA_TABLE'] != 254: + continue + suffix = f"/{route['dst_len']}" if route['dst_len'] else "" + route_str = f"route {attrs.get('RTA_DST', 'default')}{suffix} " + if attrs.get('RTA_GATEWAY'): + route_str += f"via {attrs.get('RTA_GATEWAY')} " + route_str += f"dev {ifaces[attrs['RTA_OIF']]} " + if attrs.get('RTA_PREFSRC'): + suffix = f"/{route['src_len']}" if route['src_len'] else "" + route_str += f"prefsrc {attrs.get('RTA_PREFSRC')}{suffix} " + + print(route_str) diff --git a/templates/octaviaamphoracontroller/config/octavia-healthmanager-config.json b/templates/octaviaamphoracontroller/config/octavia-healthmanager-config.json index 6dd2e42f..c4268fb2 100644 --- a/templates/octaviaamphoracontroller/config/octavia-healthmanager-config.json +++ b/templates/octaviaamphoracontroller/config/octavia-healthmanager-config.json @@ -1,5 +1,5 @@ { - "command": "/usr/local/bin/container-scripts/octavia_healthmanager_start.sh", + "command": "/usr/local/bin/container-scripts/octavia_controller_start.sh octavia-health-manager", "config_files": [ { "source": "/var/lib/config-data/merged/octavia.conf", diff --git a/templates/octaviaamphoracontroller/config/octavia-housekeeping-config.json b/templates/octaviaamphoracontroller/config/octavia-housekeeping-config.json index 5214b675..5e421fed 100644 --- a/templates/octaviaamphoracontroller/config/octavia-housekeeping-config.json +++ b/templates/octaviaamphoracontroller/config/octavia-housekeeping-config.json @@ -1,5 +1,5 @@ { - "command": "/usr/bin/octavia-housekeeping --config-file /usr/share/octavia/octavia-dist.conf --config-file /etc/octavia/octavia.conf", + "command": "/usr/local/bin/container-scripts/octavia_controller_start.sh octavia-housekeeping", "config_files": [ { "source": "/var/lib/config-data/merged/octavia.conf", diff --git a/templates/octaviaamphoracontroller/config/octavia-worker-config.json b/templates/octaviaamphoracontroller/config/octavia-worker-config.json index 4adeaad1..80e33bd6 100644 --- a/templates/octaviaamphoracontroller/config/octavia-worker-config.json +++ b/templates/octaviaamphoracontroller/config/octavia-worker-config.json @@ -1,5 +1,5 @@ { - "command": "/usr/bin/octavia-worker --config-file /usr/share/octavia/octavia-dist.conf --config-file /etc/octavia/octavia.conf", + "command": "/usr/local/bin/container-scripts/octavia_controller_start.sh octavia-worker", "config_files": [ { "source": "/var/lib/config-data/merged/octavia.conf",