Skip to content

Commit d94f6d3

Browse files
chaosi-zjuchaosi-zju
committed
local up install karmada by helm
Signed-off-by: chaosi-zju <[email protected]>
1 parent 295dee8 commit d94f6d3

File tree

3 files changed

+286
-14
lines changed

3 files changed

+286
-14
lines changed

‎.github/workflows/lint-chart.yaml

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,11 @@
11
# validate any chart changes under charts directory
2-
name: Chart Lint
2+
name: Chart Lint and Installation
33

44
env:
55
HELM_VERSION: v3.11.2
66
KUSTOMIZE_VERSION: 5.4.3
77
KIND_VERSION: v0.22.0
8-
KIND_NODE_IMAGE: kindest/node:v1.29.0
9-
K8S_VERSION: v1.29.0
8+
CLUSTER_VERSION: kindest/node:v1.29.0
109

1110
on:
1211
push:
@@ -80,13 +79,17 @@ jobs:
8079
python-version: 3.9
8180
check-latest: true
8281

82+
# Install chart-testing (https://github.com/helm/chart-testing) to judge whether charts changed.
83+
# If so, do lint testing using `ct lint` command and installation testing using `local-up-karmada-helm.sh` script.
8384
- name: Set up chart-testing
8485
uses: helm/[email protected]
8586

87+
# A necessary step in order to execute `ct list-changed` command.
8688
- name: Add dependency chart repos
8789
run: |
8890
helm repo add bitnami https://charts.bitnami.com/bitnami
8991
92+
# Judge whether charts changed by `ct list-changed` command.
9093
- name: Run chart-testing (list-changed)
9194
id: list-changed
9295
run: |
@@ -95,19 +98,15 @@ jobs:
9598
echo "changed=true" >> $GITHUB_OUTPUT
9699
fi
97100
101+
# do lint testing using `ct lint` command if charts changed.
98102
- name: Run chart-testing (lint)
99103
if: steps.list-changed.outputs.changed == 'true'
100104
run: ct lint --debug --check-version-increment=false
101105

102-
- name: Create kind cluster
103-
uses: helm/[email protected]
104-
if: steps.list-changed.outputs.changed == 'true'
105-
with:
106-
wait: 120s
107-
version: ${{ env.KIND_VERSION }}
108-
node_image: ${{ env.KIND_NODE_IMAGE }}
109-
kubectl_version: ${{ env.K8S_VERSION }}
110-
106+
# do installation testing using `local-up-karmada-helm.sh` script if charts changed.
111107
- name: Run chart-testing (install)
112108
if: steps.list-changed.outputs.changed == 'true'
113-
run: ct install --debug --helm-extra-args "--timeout 800s"
109+
run: |
110+
export KIND_VERSION=${{ env.KIND_VERSION }}
111+
export CLUSTER_VERSION=${{ env.CLUSTER_VERSION }}
112+
timeout 20m hack/local-up-karmada-helm.sh

‎charts/karmada/values.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -942,7 +942,7 @@ search:
942942
## - myRegistryKeySecretName
943943
##
944944
pullSecrets: []
945-
## @param search.resources resource quota of the search
945+
## @param search.resources resource quota of the search.
946946
resources: {}
947947
# If you do want to specify resources, uncomment the following
948948
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.

‎hack/local-up-karmada-helm.sh

Lines changed: 273 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,273 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright 2024 The Karmada Authors.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
18+
source "${REPO_ROOT}"/hack/util.sh
19+
20+
## This script deploy karmada control plane to a cluster using helm chart, and then deploy several member clusters.
21+
## Some member clusters are joined in Push mode, using karmadactl join command.
22+
## Other member clusters are joined in Pull mode, using helm chart of karmada-agent.
23+
24+
# VERSION which version Karmada you want to install
25+
VERSION=${VERSION:-"latest"}
26+
27+
# CHARTDIR the relative path of Karmada charts
28+
CHARTDIR=${CHARTDIR:-"./charts/karmada"}
29+
30+
# NEED_CREATE_KIND_CLUSTER customize whether you need to create clusters by kind, if you have clusters already, please unset these options.
31+
NEED_CREATE_KIND_CLUSTER=${NEED_CREATE_KIND_CLUSTER:-"true"}
32+
NEED_CREATE_KIND_MEMBER_CLUSTER=${NEED_CREATE_KIND_MEMBER_CLUSTER:-"true"}
33+
CLUSTER_VERSION=${CLUSTER_VERSION:-"kindest/node:v1.27.3"}
34+
35+
# KARMADA_HOST_NAME customize your cluster name and context name of karmada-host cluster, if you already has a cluster, replace it by your real name
36+
KARMADA_HOST_NAME=${KARMADA_HOST_NAME:-"karmada-host"}
37+
# KARMADA_HOST_KUBECONFIG customize your kubeconfig of karmada-host cluster, if you already has a cluster, replace it by your real kubeconfig path
38+
KARMADA_HOST_KUBECONFIG=${KARMADA_HOST_KUBECONFIG:-"${HOME}/.kube/karmada-host.config"}
39+
40+
# PUSH_MODE_MEMBERS a map which customizing your push mode member clusters
41+
# the key of the map is the cluster name / context name of your member clusters
42+
# the value of the map is the corresponding kubeconfig path of the member cluster
43+
# if you already has member clusters, replace the key with real name and replace the value with real kubeconfig path
44+
# you can add any number push mode clusters as you expect, just append this map as following examples
45+
declare -A PUSH_MODE_MEMBERS
46+
PUSH_MODE_MEMBERS["member1"]="${HOME}/.kube/member1.config"
47+
PUSH_MODE_MEMBERS["member2"]="${HOME}/.kube/member2.config"
48+
49+
# PULL_MODE_MEMBERS a map which customizing your pull mode member clusters
50+
# the key of the map is the cluster name / context name of your member clusters
51+
# the value of the map is the corresponding kubeconfig path of the member cluster
52+
# if you already has member clusters, replace the key with real name and replace the value with real kubeconfig path
53+
# you can add any number pull mode clusters as you expect, just append this map as following examples
54+
declare -A PULL_MODE_MEMBERS
55+
PULL_MODE_MEMBERS["member3"]="${HOME}/.kube/member3.config"
56+
57+
# IMAGE_FROM customize whether you need fetch images in advance, optional value are as follows:
58+
# pull-in-advance: use 'docker pull' to fetch needed images to local node in advance (in case of network trouble in automatically pulling images)
59+
# make: build karmada images from source code with latest tag and pull images (other than karmada) in advance ( in case of trying latest code)
60+
# empty or any other value: ignored, just pull image by k8s-runtime when needing
61+
IMAGE_FROM=${IMAGE_FROM:-"make"}
62+
63+
# LOAD_IMAGE_IN_ADVANCE if you fetch images in advance and you are using KinD clusters, may be you'd like to load it to KinD in advance too
64+
# if not, please unset this option
65+
LOAD_IMAGE_IN_ADVANCE=${LOAD_IMAGE_IN_ADVANCE:-"true"}
66+
67+
# INSTALL_ESTIMATOR whether install karmada-scheduler-estimator
68+
INSTALL_ESTIMATOR=${INSTALL_ESTIMATOR:-"true"}
69+
70+
# METRICS_SERVER_VERSION the target version of metrics-server
71+
METRICS_SERVER_VERSION=${METRICS_SERVER_VERSION:-"v0.6.3"}
72+
73+
# KIND_VERSION the target version of KinD
74+
KIND_VERSION=${KIND_VERSION:-"v0.22.0"}
75+
76+
# KARMADACTL_VERSION the target version of karmadactl
77+
KARMADACTL_VERSION=${KARMADACTL_VERSION:-"latest"}
78+
79+
echo "########## start installing karmada control plane ##########"
80+
81+
# 1. install KinD if comand not found and you have set NEED_CREATE_KIND_CLUSTER or NEED_CREATE_KIND_MEMBER_CLUSTER
82+
if [[ ${NEED_CREATE_KIND_CLUSTER} || ${NEED_CREATE_KIND_MEMBER_CLUSTER} ]]; then
83+
if ! util::cmd_exist kind; then
84+
echo -n "Preparing: installing kind ... "
85+
util::install_tools "sigs.k8s.io/kind" ${KIND_VERSION}
86+
fi
87+
fi
88+
89+
# 2. create KinD cluster if you set NEED_CREATE_KIND_CLUSTER
90+
if ${NEED_CREATE_KIND_CLUSTER}; then
91+
# 1.1 create karmada-host cluster
92+
kind delete clusters ${KARMADA_HOST_NAME}
93+
rm -f "${KARMADA_HOST_KUBECONFIG}"
94+
export CLUSTER_VERSION=${CLUSTER_VERSION}
95+
"${REPO_ROOT}"/hack/create-cluster.sh ${KARMADA_HOST_NAME} "${KARMADA_HOST_KUBECONFIG}"
96+
fi
97+
98+
# ALL_IMAGES all karmada component images and external images
99+
ALL_IMAGES=$(cat ${CHARTDIR}/values.yaml | grep -C 1 'repository:' | sed 's/*karmadaImageVersion/'${VERSION}'/g' | awk -F ':' '{print $2}' | sed 's/\"//g' | xargs -n3 | awk '{print $1"/"$2":"$3}')
100+
# ALL_EXTERNAL_IMAGES all external images (not include karmada component images)
101+
ALL_EXTERNAL_IMAGES=$(cat ${CHARTDIR}/values.yaml | grep -v 'karmada' | grep -C 1 'repository: ' | awk -F ':' '{print $2}' | sed 's/\"//g' | xargs -n3 | awk '{print $1"/"$2":"$3}')
102+
# AGENT_IMAGE karmada-agent image
103+
AGENT_IMAGE=$(cat ${CHARTDIR}/values.yaml | grep -C 1 'repository: karmada/karmada-agent' | sed 's/*karmadaImageVersion/'${VERSION}'/g' | awk -F ':' '{print $2}' | sed 's/\"//g' | xargs -n3 | awk '{print $1"/"$2":"$3}')
104+
105+
# 3. fetch images in advance is you set IMAGE_FROM
106+
if [ "${IMAGE_FROM}" == "pull-in-advance" ]; then
107+
## 3.1 use 'docker pull' to fetch target images to local node in advance
108+
for img in ${ALL_IMAGES}; do
109+
docker pull "${img}"
110+
done
111+
docker pull registry.k8s.io/metrics-server/metrics-server:${METRICS_SERVER_VERSION}
112+
elif [ "${IMAGE_FROM}" == "make" ]; then
113+
## 3.2 build karmada images from source code with latest tag and pull images (other than karmada) in advance
114+
for img in ${ALL_EXTERNAL_IMAGES}; do
115+
docker pull "${img}"
116+
done
117+
docker pull registry.k8s.io/metrics-server/metrics-server:${METRICS_SERVER_VERSION}
118+
119+
export VERSION=${VERSION}
120+
export REGISTRY="docker.io/karmada"
121+
make images GOOS="linux" .
122+
fi
123+
124+
# 4. load images into KinD karmada-host cluster in advance if you set LOAD_IMAGE_IN_ADVANCE
125+
if [[ ${NEED_CREATE_KIND_CLUSTER} && ${LOAD_IMAGE_IN_ADVANCE} ]]; then
126+
for img in ${ALL_IMAGES}; do
127+
kind load docker-image "${img}" --name ${KARMADA_HOST_NAME}
128+
done
129+
fi
130+
131+
# 5. this script try to deploy karmada-apiserver by host-network
132+
# so, it needs to get host-network ip (node ip) from kube-apiserver, and then add this ip to values.yaml as SANs of certificate
133+
export KUBECONFIG=${KARMADA_HOST_KUBECONFIG}
134+
HOST_IP=$(kubectl get ep kubernetes -o jsonpath='{.subsets[0].addresses[0].ip}')
135+
sed -i'' -e "/localhost/{n;s/ \"127.0.0.1/ \"${HOST_IP}\",\n&/g}" ${CHARTDIR}/values.yaml
136+
137+
# 6. install karmada at karmada-host cluster by helm
138+
# if you want to install some components, do like `--set components={"search,descheduler"}`
139+
helm install karmada -n karmada-system \
140+
--kubeconfig "${KARMADA_HOST_KUBECONFIG}" \
141+
--create-namespace \
142+
--dependency-update \
143+
--set apiServer.hostNetwork=true,components={"metricsAdapter,search,descheduler"} \
144+
${CHARTDIR}
145+
146+
# 7.verify: wait for karmada control plane ready
147+
while [[ $(kubectl get po -A | grep -c karmada-system ) -ne $(kubectl get po -n karmada-system | grep -c Running) ]]; do
148+
echo "waiting for karmada control plane ready..."; sleep 5;
149+
done
150+
kubectl get po -n karmada-system -o wide
151+
152+
# 8. export kubeconfig of karmada-apiserver to local path
153+
KARMADA_APISERVER_KUBECONFIG="${HOME}/.kube/karmada-apiserver.config"
154+
kubectl get secret -n karmada-system karmada-kubeconfig -o jsonpath={.data.kubeconfig} | base64 -d > "${KARMADA_APISERVER_KUBECONFIG}"
155+
KARMADA_APISERVER_ADDR=$(kubectl get ep karmada-apiserver -n karmada-system | tail -n 1 | awk '{print $2}')
156+
sed -i'' -e "s/karmada-apiserver.karmada-system.svc.*:5443/${KARMADA_APISERVER_ADDR}/g" "${KARMADA_APISERVER_KUBECONFIG}"
157+
158+
echo "########## end installing karmada control plane success ##########"
159+
160+
echo "########## start deploying member clusters ##########"
161+
162+
# 1. create KinD cluster if you set NEED_CREATE_KIND_MEMBER_CLUSTER
163+
if ${NEED_CREATE_KIND_MEMBER_CLUSTER}; then
164+
## 1.1. create push mode member clusters by KinD
165+
for clustername in "${!PUSH_MODE_MEMBERS[@]}"; do
166+
kind delete clusters "${clustername}"
167+
rm -f "${PUSH_MODE_MEMBERS[$clustername]}"
168+
export CLUSTER_VERSION=${CLUSTER_VERSION}
169+
"${REPO_ROOT}"/hack/create-cluster.sh "${clustername}" "${PUSH_MODE_MEMBERS[$clustername]}"
170+
171+
kind load docker-image "registry.k8s.io/metrics-server/metrics-server:${METRICS_SERVER_VERSION}" --name "${clustername}"
172+
"${REPO_ROOT}"/hack/deploy-k8s-metrics-server.sh "${PUSH_MODE_MEMBERS[$clustername]}" "${clustername}"
173+
done
174+
175+
## 1.2. create pull mode member clusters by KinD
176+
for clustername in "${!PULL_MODE_MEMBERS[@]}"; do
177+
kind delete clusters "${clustername}"
178+
rm -f "${PULL_MODE_MEMBERS[$clustername]}"
179+
export CLUSTER_VERSION=${CLUSTER_VERSION}
180+
"${REPO_ROOT}"/hack/create-cluster.sh "${clustername}" "${PULL_MODE_MEMBERS[$clustername]}"
181+
182+
kind load docker-image "registry.k8s.io/metrics-server/metrics-server:${METRICS_SERVER_VERSION}" --name "${clustername}"
183+
"${REPO_ROOT}"/hack/deploy-k8s-metrics-server.sh "${PULL_MODE_MEMBERS[$clustername]}" "${clustername}"
184+
done
185+
fi
186+
187+
# 2. load karmada-agent image into pull mode member clusters in advance if you set LOAD_IMAGE_IN_ADVANCE
188+
if [[ ${NEED_CREATE_KIND_MEMBER_CLUSTER} && ${LOAD_IMAGE_IN_ADVANCE} ]]; then
189+
for clustername in "${!PULL_MODE_MEMBERS[@]}"; do
190+
kind load docker-image "${AGENT_IMAGE}" --name "${clustername}"
191+
done
192+
fi
193+
194+
# 3. download karmadactl if command not found
195+
if ! util::cmd_exist karmadactl; then
196+
echo -n "Preparing: installing karmadactl ... "
197+
util::install_tools "github.com/karmada-io/karmada/cmd/karmadactl" ${KARMADACTL_VERSION}
198+
fi
199+
200+
declare -A ALL_MEMBERS
201+
202+
# 4. join push mode member clusters by 'karmadactl join' command
203+
for clustername in "${!PUSH_MODE_MEMBERS[@]}"; do
204+
ALL_MEMBERS[$clustername]=${PUSH_MODE_MEMBERS[$clustername]}
205+
karmadactl join "${clustername}" --kubeconfig "${KARMADA_APISERVER_KUBECONFIG}" --karmada-context karmada-apiserver --cluster-kubeconfig "${PUSH_MODE_MEMBERS[$clustername]}" --cluster-context "${clustername}"
206+
done
207+
208+
# 5. when you deploy karmada-agent by helm chart, you should manually fill in the cert of karmada-apiserver at values.yaml
209+
# so, it needs to get cert from karmada-apiserver.config for agent
210+
CA_CRT=$(cat "${KARMADA_APISERVER_KUBECONFIG}" | grep certificate-authority-data | awk -F ': ' '{print $2}' | base64 -d)
211+
AGENT_CRT=$(cat "${KARMADA_APISERVER_KUBECONFIG}" | grep client-certificate-data | awk -F ': ' '{print $2}' | base64 -d)
212+
AGENT_KEY=$(cat "${KARMADA_APISERVER_KUBECONFIG}" | grep client-key-data | awk -F ': ' '{print $2}' | base64 -d)
213+
214+
# 6. join pull mode member clusters by helm chart
215+
for clustername in "${!PULL_MODE_MEMBERS[@]}"; do
216+
clusterConfig=${PULL_MODE_MEMBERS[$clustername]}
217+
ALL_MEMBERS[$clustername]=$clusterConfig
218+
219+
MEMBER_APISERVER_ADDR=$(kubectl get ep kubernetes --kubeconfig $clusterConfig --context $clustername | tail -n 1 | awk '{print $2}')
220+
221+
helm install karmada-agent -n karmada-system \
222+
--kubeconfig "${clusterConfig}" \
223+
--kube-context "${clustername}" \
224+
--create-namespace \
225+
--dependency-update \
226+
--set installMode=agent,agent.clusterName="${clustername}",agent.clusterEndpoint=https://"${MEMBER_APISERVER_ADDR}",agent.kubeconfig.server=https://"${KARMADA_APISERVER_ADDR}",agent.kubeconfig.caCrt="${CA_CRT}",agent.kubeconfig.crt="${AGENT_CRT}",agent.kubeconfig.key="${AGENT_KEY}" \
227+
${CHARTDIR}
228+
done
229+
230+
echo "########## end deploying member clusters success ##########"
231+
232+
if ${INSTALL_ESTIMATOR}; then
233+
echo "########## start deploying karmada-scheduler-estimator ##########"
234+
235+
# as for each member cluster
236+
for clustername in "${!ALL_MEMBERS[@]}"; do
237+
# 1. you should manually fill in the cert of member cluster at values.yaml
238+
# so, it needs to get cert from member cluster kubeconfig for installing karmada-scheduler-estimator
239+
MEMBER_APISERVER=$(cat ${ALL_MEMBERS[$clustername]} | grep server: | awk -F ': ' '{print $2}')
240+
MEMBER_CA_CRT=$(cat ${ALL_MEMBERS[$clustername]} | grep certificate-authority-data | awk -F ': ' '{print $2}' | base64 -d)
241+
MEMBER_CRT=$(cat ${ALL_MEMBERS[$clustername]} | grep client-certificate-data | awk -F ': ' '{print $2}' | base64 -d)
242+
MEMBER_KEY=$(cat ${ALL_MEMBERS[$clustername]} | grep client-key-data | awk -F ': ' '{print $2}' | base64 -d)
243+
244+
# 2. install karmada-scheduler-estimator
245+
helm upgrade -i karmada-scheduler-estimator-${clustername} -n karmada-system \
246+
--kubeconfig "${KARMADA_HOST_KUBECONFIG}" \
247+
--set installMode=component,components={"schedulerEstimator"},schedulerEstimator.memberClusters[0].clusterName="${clustername}",schedulerEstimator.memberClusters[0].kubeconfig.server="${MEMBER_APISERVER}",schedulerEstimator.memberClusters[0].kubeconfig.caCrt="${MEMBER_CA_CRT}",schedulerEstimator.memberClusters[0].kubeconfig.crt="${MEMBER_CRT}",schedulerEstimator.memberClusters[0].kubeconfig.key="${MEMBER_KEY}" \
248+
${CHARTDIR}
249+
done
250+
251+
echo "########## end deploying karmada-scheduler-estimator success ##########"
252+
fi
253+
254+
# verify: wait for member cluster ready and then print member clusters
255+
export KUBECONFIG=${KARMADA_HOST_KUBECONFIG}:${KARMADA_APISERVER_KUBECONFIG}
256+
MEMBERS_NUMBER=${#ALL_MEMBERS[*]}
257+
while [[ "$(kubectl --context karmada-apiserver get clusters -o wide | grep -c "True")" -ne ${MEMBERS_NUMBER} ]]; do
258+
echo "waiting for member clusters ready..."; sleep 2;
259+
done
260+
kubectl --context karmada-apiserver get cluster -o wide
261+
262+
function print_success() {
263+
echo -e "$KARMADA_GREETING"
264+
echo "Local Karmada is running."
265+
echo -e "\nTo start using your karmada, run:"
266+
MEMBER_CLUSTER_KUBECONFIGS=$(IFS=: ; echo "${ALL_MEMBERS[*]}")
267+
echo -e " export KUBECONFIG=${KARMADA_HOST_KUBECONFIG}:${KARMADA_APISERVER_KUBECONFIG}:${MEMBER_CLUSTER_KUBECONFIGS}"
268+
echo "Please use 'kubectl config use-context ${KARMADA_HOST_NAME}/karmada-apiserver' to switch the host and control plane cluster."
269+
MEMBER_CLUSTER_NAMES=$(IFS=: ; echo "${!ALL_MEMBERS[*]}")
270+
echo "Please use 'kubectl config use-context ${MEMBER_CLUSTER_NAMES}' to switch to the different member cluster."
271+
}
272+
273+
print_success

0 commit comments

Comments
 (0)