Skip to content

Commit 50bfdaf

Browse files
cloudnullcloudnull
authored
feat: standalone CNI (rackerlabs#794)
* feat: standalone CNI This change will allow us to create some separation between what is being done in kubespray and how we deliver our clouds. The CNI implementation is now a standalone process, which allows us to deliver on being a fully k8s agnostic platform. * New scripts have been created to help facilitate the change. * An import script has been created to help existing users convert their current kube-ovn deployment into one that can be managed by helm. * Doc updates have been created to ensure that users have all of the information they need to succeed. * Updates to the documentation on setup, deployment, upgrade, and management have been added. * New helper scripts have been added for the setup * Release notes for our upcoming stable release have been added * Namespace manifests for in path services have been added Signed-off-by: Kevin Carter <[email protected]> * fix: typo old AZ referenced in ovn-setup This change updates the ovn-setup process to use az1 when an az is undefined. While our documented process has this defined correctly, the service will setup a broken AZ if the AZ is not set. Signed-off-by: Kevin Carter <[email protected]> --------- Signed-off-by: Kevin Carter <[email protected]>
1 parent 0404232 commit 50bfdaf

24 files changed

+652
-124
lines changed

‎ansible/inventory/genestack/group_vars/k8s_cluster/addons.yml

+2-3
Original file line numberDiff line numberDiff line change
@@ -166,13 +166,12 @@ cert_manager_dns_config:
166166
- "8.8.8.8"
167167

168168
cert_manager_controller_extra_args:
169-
- "--feature-gates=ExperimentalGatewayAPISupport=true"
170-
# - "--enable-gateway-api" # Not available until cert manager v1.15
169+
- "--enable-gateway-api"
171170
# - "--dns01-recursive-nameservers-only=true"
172171
# - "--dns01-recursive-nameservers=1.1.1.1:53,8.8.8.8:53"
173172

174173
# MetalLB deployment
175-
metallb_enabled: true
174+
metallb_enabled: false
176175
metallb_namespace: metallb-system
177176
metallb_speaker_enabled: "{{ metallb_enabled }}"
178177
# metallb_version: v0.13.9

‎ansible/inventory/genestack/group_vars/k8s_cluster/k8s-cluster.yml

+16-3
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,9 @@ credentials_dir: "{{ inventory_dir }}/credentials"
6565
# kube_webhook_authorization_url: https://...
6666
# kube_webhook_authorization_url_skip_tls_verify: false
6767

68-
# Choose network plugin (cilium, calico, kube-ovn, weave or flannel. Use cni for generic cni plugin)
68+
# Choose network plugin (cilium, calico, kube-ovn, weave, flannel, or none. Use cni for generic cni plugin)
6969
# Can also be set to 'cloud', which lets the cloud provider setup appropriate routing
70-
kube_network_plugin: kube-ovn
70+
kube_network_plugin: none
7171

7272
# Setting multi_networking to true will install Multus: https://github.com/k8snetworkplumbingwg/multus-cni
7373
kube_network_plugin_multus: false
@@ -384,5 +384,18 @@ auto_renew_certificates: false
384384
# First Monday of each month
385385
# auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:{{ groups['kube_control_plane'].index(inventory_hostname) }}0:00"
386386

387-
# kubeadm patches path
388387
kubeadm_patches: []
388+
# - target: kube-controller-manager
389+
# type: strategic(default)|json|merge
390+
# patch:
391+
# metadata:
392+
# annotations:
393+
# prometheus.io/scrape: 'true'
394+
# prometheus.io/port: '10257'
395+
# - target: kube-scheduler
396+
# type: strategic(default)|json|merge
397+
# patch:
398+
# metadata:
399+
# annotations:
400+
# prometheus.io/scrape: 'true'
401+
# prometheus.io/port: '10259'

‎ansible/inventory/genestack/group_vars/k8s_cluster/k8s-net-kube-ovn.yml

-69
This file was deleted.

‎ansible/inventory/genestack/patches/kube-controller-manager+merge.yaml

-8
This file was deleted.

‎ansible/inventory/genestack/patches/kube-scheduler+merge.yaml

-8
This file was deleted.

‎base-helm-configs/kube-ovn/kube-ovn-helm-overrides.yaml

+192
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
# Default values for kubeovn.
2+
# This is a YAML-formatted file.
3+
# Declare variables to be passed into your templates.
4+
global:
5+
registry:
6+
address: ghcr.io/rackerlabs
7+
imagePullSecrets: []
8+
images:
9+
kubeovn:
10+
repository: kube-ovn
11+
dpdkRepository: kube-ovn-dpdk
12+
vpcRepository: vpc-nat-gateway
13+
# Change "tag" when PR https://github.com/kubeovn/kube-ovn/pull/5005 is merged
14+
tag: v1.12.32-gc-disable
15+
support_arm: true
16+
thirdparty: true
17+
18+
image:
19+
pullPolicy: IfNotPresent
20+
21+
replicaCount: 3
22+
23+
namespace: kube-system
24+
25+
MASTER_NODES: ""
26+
MASTER_NODES_LABEL: "kube-ovn/role=master"
27+
28+
networking:
29+
# NET_STACK could be dual_stack, ipv4, ipv6
30+
NET_STACK: ipv4
31+
ENABLE_SSL: false
32+
# network type could be geneve or vlan
33+
NETWORK_TYPE: geneve
34+
# tunnel type could be geneve, vxlan or stt
35+
TUNNEL_TYPE: geneve
36+
IFACE: "br-overlay"
37+
DPDK_TUNNEL_IFACE: "br-phy"
38+
EXCLUDE_IPS: ""
39+
POD_NIC_TYPE: "veth-pair"
40+
vlan:
41+
PROVIDER_NAME: "provider"
42+
VLAN_INTERFACE_NAME: "br-overlay"
43+
VLAN_NAME: "ovn-vlan"
44+
VLAN_ID: "100"
45+
EXCHANGE_LINK_NAME: false
46+
ENABLE_EIP_SNAT: true
47+
DEFAULT_SUBNET: "ovn-default"
48+
DEFAULT_VPC: "ovn-cluster"
49+
NODE_SUBNET: "join"
50+
ENABLE_ECMP: false
51+
ENABLE_METRICS: true
52+
# comma-separated string of nodelocal DNS ip addresses
53+
NODE_LOCAL_DNS_IP: ""
54+
PROBE_INTERVAL: 180000
55+
OVN_NORTHD_PROBE_INTERVAL: 5000
56+
OVN_LEADER_PROBE_INTERVAL: 5
57+
OVN_REMOTE_PROBE_INTERVAL: 10000
58+
OVN_REMOTE_OPENFLOW_INTERVAL: 180
59+
OVN_NORTHD_N_THREADS: 1
60+
ENABLE_COMPACT: false
61+
62+
func:
63+
ENABLE_LB: true
64+
ENABLE_NP: true
65+
ENABLE_EXTERNAL_VPC: true
66+
HW_OFFLOAD: false
67+
ENABLE_LB_SVC: false
68+
ENABLE_KEEP_VM_IP: true
69+
LS_DNAT_MOD_DL_DST: true
70+
LS_CT_SKIP_DST_LPORT_IPS: true
71+
CHECK_GATEWAY: true
72+
LOGICAL_GATEWAY: false
73+
ENABLE_BIND_LOCAL_IP: true
74+
SECURE_SERVING: false
75+
U2O_INTERCONNECTION: false
76+
ENABLE_TPROXY: false
77+
ENABLE_IC: false
78+
ENABLE_NAT_GW: true
79+
ENABLE_OVN_IPSEC: false
80+
ENABLE_ANP: false
81+
SET_VXLAN_TX_OFF: false
82+
OVSDB_CON_TIMEOUT: 3
83+
OVSDB_INACTIVITY_TIMEOUT: 10
84+
ENABLE_LIVE_MIGRATION_OPTIMIZE: true
85+
86+
ipv4:
87+
POD_CIDR: "10.236.0.0/14"
88+
POD_GATEWAY: "10.236.0.1"
89+
SVC_CIDR: "10.233.0.0/18"
90+
JOIN_CIDR: "100.64.0.0/16"
91+
PINGER_EXTERNAL_ADDRESS: "208.67.222.222"
92+
PINGER_EXTERNAL_DOMAIN: "opendns.com."
93+
94+
ipv6:
95+
POD_CIDR: "fd00:10:16::/112"
96+
POD_GATEWAY: "fd00:10:16::1"
97+
SVC_CIDR: "fd00:10:96::/112"
98+
JOIN_CIDR: "fd00:100:64::/112"
99+
PINGER_EXTERNAL_ADDRESS: "2620:119:35::35"
100+
PINGER_EXTERNAL_DOMAIN: "opendns.com."
101+
102+
dual_stack:
103+
POD_CIDR: "10.236.0.0/14,fd00:10:16::/112"
104+
POD_GATEWAY: "10.236.0.1,fd00:10:16::1"
105+
SVC_CIDR: "10.233.0.0/18,fd00:10:96::/112"
106+
JOIN_CIDR: "100.64.0.0/16,fd00:100:64::/112"
107+
PINGER_EXTERNAL_ADDRESS: "208.67.222.222,2620:119:35::35"
108+
PINGER_EXTERNAL_DOMAIN: "opendns.com."
109+
110+
performance:
111+
GC_INTERVAL: 0
112+
INSPECT_INTERVAL: 20
113+
OVS_VSCTL_CONCURRENCY: 100
114+
115+
debug:
116+
ENABLE_MIRROR: false
117+
MIRROR_IFACE: "mirror0"
118+
119+
cni_conf:
120+
CNI_CONFIG_PRIORITY: "01"
121+
CNI_CONF_DIR: "/etc/cni/net.d"
122+
CNI_BIN_DIR: "/opt/cni/bin"
123+
CNI_CONF_FILE: "/kube-ovn/01-kube-ovn.conflist"
124+
LOCAL_BIN_DIR: "/usr/local/bin"
125+
MOUNT_LOCAL_BIN_DIR: false
126+
127+
kubelet_conf:
128+
KUBELET_DIR: "/var/lib/kubelet"
129+
130+
log_conf:
131+
LOG_DIR: "/var/log"
132+
133+
OPENVSWITCH_DIR: "/etc/origin/openvswitch"
134+
OVN_DIR: "/etc/origin/ovn"
135+
DISABLE_MODULES_MANAGEMENT: false
136+
137+
nameOverride: ""
138+
fullnameOverride: ""
139+
140+
# hybrid dpdk
141+
HYBRID_DPDK: false
142+
HUGEPAGE_SIZE_TYPE: hugepages-2Mi # Default
143+
HUGEPAGES: 1Gi
144+
145+
# DPDK
146+
DPDK: false
147+
DPDK_VERSION: "19.11"
148+
DPDK_CPU: "1000m" # Default CPU configuration
149+
DPDK_MEMORY: "2Gi" # Default Memory configuration
150+
151+
ovn-central:
152+
requests:
153+
cpu: "300m"
154+
memory: "200Mi"
155+
limits:
156+
cpu: "3"
157+
memory: "4Gi"
158+
ovs-ovn:
159+
requests:
160+
cpu: "200m"
161+
memory: "200Mi"
162+
limits:
163+
cpu: "2"
164+
memory: "1000Mi"
165+
kube-ovn-controller:
166+
requests:
167+
cpu: "200m"
168+
memory: "200Mi"
169+
limits:
170+
cpu: "1000m"
171+
memory: "1Gi"
172+
kube-ovn-cni:
173+
requests:
174+
cpu: "100m"
175+
memory: "100Mi"
176+
limits:
177+
cpu: "1000m"
178+
memory: "1Gi"
179+
kube-ovn-pinger:
180+
requests:
181+
cpu: "100m"
182+
memory: "100Mi"
183+
limits:
184+
cpu: "200m"
185+
memory: "400Mi"
186+
kube-ovn-monitor:
187+
requests:
188+
cpu: "200m"
189+
memory: "200Mi"
190+
limits:
191+
cpu: "200m"
192+
memory: "200Mi"

‎base-helm-configs/metallb/metallb-helm-overrides.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
---

‎base-kustomize/ovn/ovn-setup.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,9 @@ data:
7575
ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings="${OVN_BRIDGE_MAPPINGS}"
7676
fi
7777
if [[ "${OVN_GW_ENABLED:-disabled}" == "enabled" ]]; then
78-
ovs-vsctl set Open_vSwitch . external-ids:ovn-cms-options="enable-chassis-as-gw,availability-zones=${OVN_AVAILABILITY_ZONES:-nova}"
78+
ovs-vsctl set Open_vSwitch . external-ids:ovn-cms-options="enable-chassis-as-gw,availability-zones=${OVN_AVAILABILITY_ZONES:-az1}"
7979
else
80-
ovs-vsctl set Open_vSwitch . external-ids:ovn-cms-options="availability-zones=${OVN_AVAILABILITY_ZONES:-nova}"
80+
ovs-vsctl set Open_vSwitch . external-ids:ovn-cms-options="availability-zones=${OVN_AVAILABILITY_ZONES:-az1}"
8181
fi
8282
if [[ -z $(ovs-vsctl get-manager) ]]; then
8383
ovs-vsctl --id @manager create Manager 'target="ptcp:6640:127.0.0.1"' -- add Open_vSwitch . manager_options @manager

‎bin/install-kube-ovn.sh

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
# shellcheck disable=SC2124,SC2145,SC2294
3+
4+
GLOBAL_OVERRIDES_DIR="/etc/genestack/helm-configs/global_overrides"
5+
SERVICE_CONFIG_DIR="/etc/genestack/helm-configs/kube-ovn"
6+
BASE_OVERRIDES="/opt/genestack/base-helm-configs/kube-ovn/kube-ovn-helm-overrides.yaml"
7+
KUBE_OVN_VERSION="v1.12.30"
8+
MASTER_NODES=$(kubectl get nodes -l kube-ovn/role=master -o json | jq -r '[.items[].status.addresses[] | select(.type == "InternalIP") | .address] | join(",")' | sed 's/,/\\,/g')
9+
MASTER_NODE_COUNT=$(kubectl get nodes -l kube-ovn/role=master -o json | jq -r '.items[].status.addresses[] | select(.type=="InternalIP") | .address' | wc -l)
10+
11+
if [ "${MASTER_NODE_COUNT}" -eq 0 ]; then
12+
echo "No master nodes found"
13+
echo "Be sure to label your master nodes with kube-ovn/role=master before running this script"
14+
echo "Exiting"
15+
exit 1
16+
fi
17+
18+
helm repo add kubeovn https://kubeovn.github.io/kube-ovn
19+
helm repo update
20+
21+
HELM_CMD="helm upgrade --install kube-ovn kubeovn/kube-ovn \
22+
--version ${KUBE_OVN_VERSION} \
23+
--namespace=kube-system \
24+
--set MASTER_NODES=\"${MASTER_NODES}\" \
25+
--set replicaCount=${MASTER_NODE_COUNT}"
26+
27+
HELM_CMD+=" -f ${BASE_OVERRIDES}"
28+
29+
for dir in "$GLOBAL_OVERRIDES_DIR" "$SERVICE_CONFIG_DIR"; do
30+
if compgen -G "${dir}/*.yaml" > /dev/null; then
31+
for yaml_file in "${dir}"/*.yaml; do
32+
# Avoid re-adding the base override file if present in the service directory
33+
if [ "${yaml_file}" != "${BASE_OVERRIDES}" ]; then
34+
HELM_CMD+=" -f ${yaml_file}"
35+
fi
36+
done
37+
fi
38+
done
39+
40+
HELM_CMD+=" $@"
41+
42+
echo "Executing Helm command:"
43+
echo "${HELM_CMD}"
44+
eval "${HELM_CMD}"

0 commit comments

Comments
 (0)