Skip to content

Commit 67e941f

Browse files
authored
cks: fix k8s version upgrade (apache#6513)
Fixes apache#6514 On latest systemvm template used for CKS /usr/sbin is not present in the $PATH for normal user used during upgrade. This leads to failure for blkid command. Due to this during k8s version upgrade ISO is not being able to mount on the k8s cluster VMs and upgrade process is not carried out. This PR fixes mounting of k8s version ISO and also returns failure for script when ISO mounting is failed. Same failure is not seen during deployment of the cluster because setup-kube-system workflow is executed as ROOT user and it has a different value for $PATH. From /etc/login.defs: ENV_SUPATH PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ENV_PATH PATH=/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games Signed-off-by: Abhishek Kumar <[email protected]>
1 parent 9e5cda5 commit 67e941f

File tree

4 files changed

+140
-2
lines changed

4 files changed

+140
-2
lines changed

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java

+4-1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import java.util.List;
2323

2424
import org.apache.commons.collections.CollectionUtils;
25+
import org.apache.commons.lang3.StringUtils;
2526
import org.apache.log4j.Level;
2627

2728
import com.cloud.hypervisor.Hypervisor;
@@ -36,7 +37,6 @@
3637
import com.cloud.utils.Pair;
3738
import com.cloud.utils.exception.CloudRuntimeException;
3839
import com.cloud.utils.ssh.SshHelper;
39-
import org.apache.commons.lang3.StringUtils;
4040

4141
public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorker {
4242

@@ -122,6 +122,9 @@ private void upgradeKubernetesClusterNodes() {
122122
logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to get control Kubernetes node on VM : %s in ready state", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null);
123123
}
124124
}
125+
if (!KubernetesClusterUtil.clusterNodeVersionMatches(upgradeVersion.getSemanticVersion(), i==0, publicIpAddress, sshPort, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile(), hostName)) {
126+
logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to get Kubernetes node on VM : %s upgraded to version %s", kubernetesCluster.getName(), vm.getDisplayName(), upgradeVersion.getSemanticVersion()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null);
127+
}
125128
if (LOGGER.isInfoEnabled()) {
126129
LOGGER.info(String.format("Successfully upgraded node on VM %s in Kubernetes cluster %s with Kubernetes version(%s) ID: %s",
127130
vm.getDisplayName(), kubernetesCluster.getName(), upgradeVersion.getSemanticVersion(), upgradeVersion.getUuid()));

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java

+37-1
Original file line numberDiff line numberDiff line change
@@ -32,19 +32,21 @@
3232
import javax.net.ssl.TrustManager;
3333

3434
import org.apache.cloudstack.utils.security.SSLUtils;
35+
import org.apache.commons.lang3.StringUtils;
3536
import org.apache.log4j.Logger;
3637

3738
import com.cloud.kubernetes.cluster.KubernetesCluster;
3839
import com.cloud.uservm.UserVm;
3940
import com.cloud.utils.Pair;
4041
import com.cloud.utils.nio.TrustAllManager;
4142
import com.cloud.utils.ssh.SshHelper;
42-
import org.apache.commons.lang3.StringUtils;
4343

4444
public class KubernetesClusterUtil {
4545

4646
protected static final Logger LOGGER = Logger.getLogger(KubernetesClusterUtil.class);
4747

48+
public static final String CLUSTER_NODE_VERSION_COMMAND = "sudo /opt/bin/kubectl version --short";
49+
4850
public static boolean isKubernetesClusterNodeReady(final KubernetesCluster kubernetesCluster, String ipAddress, int port,
4951
String user, File sshKeyFile, String nodeName) throws Exception {
5052
Pair<Boolean, String> result = SshHelper.sshExecute(ipAddress, port,
@@ -324,4 +326,38 @@ public static String generateClusterHACertificateKey(final KubernetesCluster kub
324326
}
325327
return token.toString().substring(0, 64);
326328
}
329+
330+
public static boolean clusterNodeVersionMatches(final String version, boolean isControlNode,
331+
final String ipAddress, final int port,
332+
final String user, final File sshKeyFile,
333+
final String hostName) {
334+
Pair<Boolean, String> result = null;
335+
try {
336+
result = SshHelper.sshExecute(
337+
ipAddress, port,
338+
user, sshKeyFile, null,
339+
CLUSTER_NODE_VERSION_COMMAND,
340+
10000, 10000, 20000);
341+
} catch (Exception e) {
342+
if (LOGGER.isDebugEnabled()) {
343+
LOGGER.debug(String.format("Failed to retrieve Kubernetes version from cluster node : %s due to exception", hostName), e);
344+
}
345+
return false;
346+
}
347+
if (Boolean.FALSE.equals(result.first()) || StringUtils.isBlank(result.second())) {
348+
return false;
349+
}
350+
String response = result.second();
351+
boolean clientVersionPresent = false;
352+
boolean serverVersionPresent = false;
353+
for (String line : response.split("\n")) {
354+
if (line.contains("Client Version") && line.contains(String.format("v%s", version))) {
355+
clientVersionPresent = true;
356+
}
357+
if (isControlNode && line.contains("Server Version") && line.contains(String.format("v%s", version))) {
358+
serverVersionPresent = true;
359+
}
360+
}
361+
return clientVersionPresent && (!isControlNode || serverVersionPresent);
362+
}
327363
}

plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh

+6
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@ if [ $# -gt 3 ]; then
3737
fi
3838

3939
export PATH=$PATH:/opt/bin
40+
if [[ "$PATH" != *:/usr/sbin && "$PATH" != *:/usr/sbin:* ]]; then
41+
export PATH=$PATH:/usr/sbin
42+
fi
4043

4144
ISO_MOUNT_DIR=/mnt/k8sdisk
4245
BINARIES_DIR=${ISO_MOUNT_DIR}/
@@ -149,4 +152,7 @@ if [ -d "$BINARIES_DIR" ]; then
149152
if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then
150153
eject "${iso_drive_path}"
151154
fi
155+
else
156+
echo "ERROR: Unable to access Binaries directory for upgrade version ${UPGRADE_VERSION}"
157+
exit 1
152158
fi
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
package com.cloud.kubernetes.cluster.utils;
18+
19+
import java.io.File;
20+
21+
import org.junit.Assert;
22+
import org.junit.Test;
23+
import org.junit.runner.RunWith;
24+
import org.mockito.Mockito;
25+
import org.powermock.api.mockito.PowerMockito;
26+
import org.powermock.core.classloader.annotations.PrepareForTest;
27+
import org.powermock.modules.junit4.PowerMockRunner;
28+
29+
import com.cloud.utils.Pair;
30+
import com.cloud.utils.ssh.SshHelper;
31+
32+
@RunWith(PowerMockRunner.class)
33+
@PrepareForTest(SshHelper.class)
34+
public class KubernetesClusterUtilTest {
35+
String ipAddress = "10.1.1.1";
36+
int port = 2222;
37+
String user = "user";
38+
File sshKeyFile = Mockito.mock(File.class);
39+
String hostName = "host";
40+
41+
private void mockSshHelperExecuteThrowAndTestVersionMatch() {
42+
try {
43+
Mockito.when(SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, KubernetesClusterUtil.CLUSTER_NODE_VERSION_COMMAND, 10000, 10000, 20000)).thenThrow(Exception.class);
44+
} catch (Exception e) {
45+
Assert.fail(String.format("Exception: %s", e.getMessage()));
46+
}
47+
Assert.assertFalse(KubernetesClusterUtil.clusterNodeVersionMatches("1.24.0", false, ipAddress, port, user, sshKeyFile, hostName));
48+
}
49+
50+
private void mockSshHelperExecuteAndTestVersionMatch(boolean status, String response, boolean isControlNode, boolean expectedResult) {
51+
try {
52+
Mockito.when(SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, KubernetesClusterUtil.CLUSTER_NODE_VERSION_COMMAND, 10000, 10000, 20000)).thenReturn(new Pair<>(status, response));
53+
} catch (Exception e) {
54+
Assert.fail(String.format("Exception: %s", e.getMessage()));
55+
}
56+
boolean result = KubernetesClusterUtil.clusterNodeVersionMatches("1.24.0", isControlNode, ipAddress, port, user, sshKeyFile, hostName);
57+
Assert.assertEquals(expectedResult, result);
58+
}
59+
60+
@Test
61+
public void testClusterNodeVersionMatches() {
62+
PowerMockito.mockStatic(SshHelper.class);
63+
String v1233WorkerNodeOutput = "Client Version: v1.23.3\n" +
64+
"The connection to the server localhost:8080 was refused - did you specify the right host or port?";
65+
String v1240WorkerNodeOutput = "Client Version: v1.24.0\n" +
66+
"Kustomize Version: v4.5.4\n" +
67+
"The connection to the server localhost:8080 was refused - did you specify the right host or port?";
68+
String v1240ControlNodeOutput = "Client Version: v1.24.0\n" +
69+
"Kustomize Version: v4.5.4\n" +
70+
"Server Version: v1.24.0";
71+
mockSshHelperExecuteAndTestVersionMatch(true, v1240WorkerNodeOutput, false, true);
72+
73+
mockSshHelperExecuteAndTestVersionMatch(true, v1240ControlNodeOutput, true, true);
74+
75+
mockSshHelperExecuteAndTestVersionMatch(true, v1240WorkerNodeOutput, true, false);
76+
77+
mockSshHelperExecuteAndTestVersionMatch(false, v1240WorkerNodeOutput, false, false);
78+
79+
mockSshHelperExecuteAndTestVersionMatch(true, v1233WorkerNodeOutput, false, false);
80+
81+
mockSshHelperExecuteAndTestVersionMatch(true, "Client Version: v1.24.0\n" +
82+
"Kustomize Version: v4.5.4\n" +
83+
"Server Version: v1.23.0", true, false);
84+
85+
mockSshHelperExecuteAndTestVersionMatch(true, null, false, false);
86+
87+
mockSshHelperExecuteAndTestVersionMatch(false, "-\n-", false, false);
88+
89+
mockSshHelperExecuteAndTestVersionMatch(false, "1.24.0", false, false);
90+
91+
mockSshHelperExecuteThrowAndTestVersionMatch();
92+
}
93+
}

0 commit comments

Comments
 (0)