Skip to content

Commit 539d7e1

Browse files
authored
Merge pull request apache#2493 from shapeblue/fixmaintenance
CLOUDSTACK-10326: Prevent hosts fall into Maintenance when there are running VMs on it
2 parents 39471c8 + faf2a77 commit 539d7e1

File tree

5 files changed

+287
-6
lines changed

5 files changed

+287
-6
lines changed

engine/schema/src/com/cloud/vm/dao/VMInstanceDao.java

+2
Original file line numberDiff line numberDiff line change
@@ -150,4 +150,6 @@ public interface VMInstanceDao extends GenericDao<VMInstanceVO, Long>, StateDao<
150150
VMInstanceVO findVMByHostNameInZone(String hostName, long zoneId);
151151

152152
boolean isPowerStateUpToDate(long instanceId);
153+
154+
List<VMInstanceVO> listNonMigratingVmsByHostEqualsLastHost(long hostId);
153155
}

engine/schema/src/com/cloud/vm/dao/VMInstanceDaoImpl.java

+15
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
9393
protected GenericSearchBuilder<VMInstanceVO, String> DistinctHostNameSearch;
9494
protected SearchBuilder<VMInstanceVO> HostAndStateSearch;
9595
protected SearchBuilder<VMInstanceVO> StartingWithNoHostSearch;
96+
protected SearchBuilder<VMInstanceVO> NotMigratingSearch;
9697

9798
@Inject
9899
ResourceTagDao _tagsDao;
@@ -280,6 +281,11 @@ protected void init() {
280281
DistinctHostNameSearch.join("nicSearch", nicSearch, DistinctHostNameSearch.entity().getId(), nicSearch.entity().getInstanceId(), JoinBuilder.JoinType.INNER);
281282
DistinctHostNameSearch.done();
282283

284+
NotMigratingSearch = createSearchBuilder();
285+
NotMigratingSearch.and("host", NotMigratingSearch.entity().getHostId(), Op.EQ);
286+
NotMigratingSearch.and("lastHost", NotMigratingSearch.entity().getLastHostId(), Op.EQ);
287+
NotMigratingSearch.and("state", NotMigratingSearch.entity().getState(), Op.NEQ);
288+
NotMigratingSearch.done();
283289
}
284290

285291
@Override
@@ -304,6 +310,15 @@ public List<VMInstanceVO> listByHostId(long hostid) {
304310
return listBy(sc);
305311
}
306312

313+
@Override
314+
public List<VMInstanceVO> listNonMigratingVmsByHostEqualsLastHost(long hostId) {
315+
SearchCriteria<VMInstanceVO> sc = NotMigratingSearch.create();
316+
sc.setParameters("host", hostId);
317+
sc.setParameters("lastHost", hostId);
318+
sc.setParameters("state", State.Migrating);
319+
return listBy(sc);
320+
}
321+
307322
@Override
308323
public List<VMInstanceVO> listByZoneId(long zoneId) {
309324
SearchCriteria<VMInstanceVO> sc = AllFieldsSearch.create();

server/src/com/cloud/resource/ResourceManagerImpl.java

+70-5
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import javax.inject.Inject;
3131
import javax.naming.ConfigurationException;
3232

33+
import com.cloud.vm.dao.UserVmDetailsDao;
3334
import org.apache.commons.lang.ObjectUtils;
3435
import org.apache.log4j.Logger;
3536
import org.springframework.stereotype.Component;
@@ -54,6 +55,8 @@
5455
import com.cloud.agent.AgentManager;
5556
import com.cloud.agent.api.Answer;
5657
import com.cloud.agent.api.Command;
58+
import com.cloud.agent.api.GetVncPortCommand;
59+
import com.cloud.agent.api.GetVncPortAnswer;
5760
import com.cloud.agent.api.GetGPUStatsAnswer;
5861
import com.cloud.agent.api.GetGPUStatsCommand;
5962
import com.cloud.agent.api.GetHostStatsAnswer;
@@ -252,6 +255,8 @@ public void setDiscoverers(final List<? extends Discoverer> discoverers) {
252255
private ConfigurationManager _configMgr;
253256
@Inject
254257
private ClusterVSMMapDao _clusterVSMMapDao;
258+
@Inject
259+
private UserVmDetailsDao userVmDetailsDao;
255260

256261
private final long _nodeId = ManagementServerNode.getManagementServerId();
257262

@@ -1287,6 +1292,68 @@ public Host maintain(final PrepareForMaintenanceCmd cmd) {
12871292
}
12881293
}
12891294

1295+
/**
1296+
* Add VNC details as user VM details for each VM in 'vms' (KVM hosts only)
1297+
*/
1298+
protected void setKVMVncAccess(long hostId, List<VMInstanceVO> vms) {
1299+
for (VMInstanceVO vm : vms) {
1300+
GetVncPortAnswer vmVncPortAnswer = (GetVncPortAnswer) _agentMgr.easySend(hostId, new GetVncPortCommand(vm.getId(), vm.getInstanceName()));
1301+
if (vmVncPortAnswer != null) {
1302+
userVmDetailsDao.addDetail(vm.getId(), "kvm.vnc.address", vmVncPortAnswer.getAddress(), true);
1303+
userVmDetailsDao.addDetail(vm.getId(), "kvm.vnc.port", String.valueOf(vmVncPortAnswer.getPort()), true);
1304+
}
1305+
}
1306+
}
1307+
1308+
/**
1309+
* Configure VNC access for host VMs which have failed migrating to another host while trying to enter Maintenance mode
1310+
*/
1311+
protected void configureVncAccessForKVMHostFailedMigrations(HostVO host, List<VMInstanceVO> failedMigrations) {
1312+
if (host.getHypervisorType().equals(HypervisorType.KVM)) {
1313+
_agentMgr.pullAgentOutMaintenance(host.getId());
1314+
setKVMVncAccess(host.getId(), failedMigrations);
1315+
_agentMgr.pullAgentToMaintenance(host.getId());
1316+
}
1317+
}
1318+
1319+
/**
1320+
* Set host into ErrorInMaintenance state, as errors occurred during VM migrations. Do the following:
1321+
* - Cancel scheduled migrations for those which have already failed
1322+
* - Configure VNC access for VMs (KVM hosts only)
1323+
*/
1324+
protected boolean setHostIntoErrorInMaintenance(HostVO host, List<VMInstanceVO> failedMigrations) throws NoTransitionException {
1325+
s_logger.debug("Unable to migrate " + failedMigrations.size() + " VM(s) from host " + host.getUuid());
1326+
_haMgr.cancelScheduledMigrations(host);
1327+
configureVncAccessForKVMHostFailedMigrations(host, failedMigrations);
1328+
resourceStateTransitTo(host, ResourceState.Event.UnableToMigrate, _nodeId);
1329+
return false;
1330+
}
1331+
1332+
/**
1333+
* Safely transit host into Maintenance mode
1334+
*/
1335+
protected boolean setHostIntoMaintenance(HostVO host) throws NoTransitionException {
1336+
s_logger.debug("Host " + host.getUuid() + " entering in Maintenance");
1337+
resourceStateTransitTo(host, ResourceState.Event.InternalEnterMaintenance, _nodeId);
1338+
ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(),
1339+
EventVO.LEVEL_INFO, EventTypes.EVENT_MAINTENANCE_PREPARE,
1340+
"completed maintenance for host " + host.getId(), 0);
1341+
return true;
1342+
}
1343+
1344+
/**
1345+
* Return true if host goes into Maintenance mode, only when:
1346+
* - No Running, Migrating or Failed migrations (host_id = last_host_id) for the host
1347+
*/
1348+
protected boolean isHostInMaintenance(HostVO host, List<VMInstanceVO> runningVms, List<VMInstanceVO> migratingVms, List<VMInstanceVO> failedMigrations) throws NoTransitionException {
1349+
if (CollectionUtils.isEmpty(runningVms) && CollectionUtils.isEmpty(migratingVms)) {
1350+
return CollectionUtils.isEmpty(failedMigrations) ?
1351+
setHostIntoMaintenance(host) :
1352+
setHostIntoErrorInMaintenance(host, failedMigrations);
1353+
}
1354+
return false;
1355+
}
1356+
12901357
@Override
12911358
public boolean checkAndMaintain(final long hostId) {
12921359
boolean hostInMaintenance = false;
@@ -1296,11 +1363,9 @@ public boolean checkAndMaintain(final long hostId) {
12961363
if (host.getType() != Host.Type.Storage) {
12971364
final List<VMInstanceVO> vos = _vmDao.listByHostId(hostId);
12981365
final List<VMInstanceVO> vosMigrating = _vmDao.listVmsMigratingFromHost(hostId);
1299-
if (vos.isEmpty() && vosMigrating.isEmpty()) {
1300-
resourceStateTransitTo(host, ResourceState.Event.InternalEnterMaintenance, _nodeId);
1301-
hostInMaintenance = true;
1302-
ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_MAINTENANCE_PREPARE, "completed maintenance for host " + hostId, 0);
1303-
}
1366+
final List<VMInstanceVO> failedVmMigrations = _vmDao.listNonMigratingVmsByHostEqualsLastHost(hostId);
1367+
1368+
hostInMaintenance = isHostInMaintenance(host, vos, vosMigrating, failedVmMigrations);
13041369
}
13051370
} catch (final NoTransitionException e) {
13061371
s_logger.debug("Cannot transmit host " + host.getId() + "to Maintenance state", e);

server/src/com/cloud/servlet/ConsoleProxyServlet.java

+9-1
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import javax.servlet.http.HttpServletResponse;
3636
import javax.servlet.http.HttpSession;
3737

38+
import com.cloud.resource.ResourceState;
3839
import org.apache.commons.codec.binary.Base64;
3940
import org.apache.log4j.Logger;
4041
import org.springframework.stereotype.Component;
@@ -418,7 +419,14 @@ private String composeConsoleAccessUrl(String rootUrl, VirtualMachine vm, HostVO
418419
StringBuffer sb = new StringBuffer(rootUrl);
419420
String host = hostVo.getPrivateIpAddress();
420421

421-
Pair<String, Integer> portInfo = _ms.getVncPort(vm);
422+
Pair<String, Integer> portInfo;
423+
if (hostVo.getResourceState().equals(ResourceState.ErrorInMaintenance)) {
424+
UserVmDetailVO detailAddress = _userVmDetailsDao.findDetail(vm.getId(), "kvm.vnc.address");
425+
UserVmDetailVO detailPort = _userVmDetailsDao.findDetail(vm.getId(), "kvm.vnc.port");
426+
portInfo = new Pair<>(detailAddress.getValue(), Integer.valueOf(detailPort.getValue()));
427+
} else {
428+
portInfo = _ms.getVncPort(vm);
429+
}
422430
if (s_logger.isDebugEnabled())
423431
s_logger.debug("Port info " + portInfo.first());
424432

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package com.cloud.resource;
19+
20+
import com.cloud.agent.AgentManager;
21+
import com.cloud.agent.api.GetVncPortAnswer;
22+
import com.cloud.agent.api.GetVncPortCommand;
23+
import com.cloud.capacity.dao.CapacityDao;
24+
import com.cloud.event.ActionEventUtils;
25+
import com.cloud.ha.HighAvailabilityManager;
26+
import com.cloud.host.Host;
27+
import com.cloud.host.HostVO;
28+
import com.cloud.host.dao.HostDao;
29+
import com.cloud.hypervisor.Hypervisor;
30+
import com.cloud.storage.StorageManager;
31+
import com.cloud.utils.fsm.NoTransitionException;
32+
import com.cloud.vm.VMInstanceVO;
33+
import com.cloud.vm.dao.UserVmDetailsDao;
34+
import com.cloud.vm.dao.VMInstanceDao;
35+
import org.junit.Assert;
36+
import org.junit.Before;
37+
import org.junit.Test;
38+
import org.junit.runner.RunWith;
39+
import org.mockito.BDDMockito;
40+
import org.mockito.InjectMocks;
41+
import org.mockito.Mock;
42+
import org.mockito.MockitoAnnotations;
43+
import org.mockito.Spy;
44+
import org.powermock.api.mockito.PowerMockito;
45+
import org.powermock.core.classloader.annotations.PrepareForTest;
46+
import org.powermock.modules.junit4.PowerMockRunner;
47+
48+
import java.util.ArrayList;
49+
import java.util.Arrays;
50+
import java.util.List;
51+
52+
import static com.cloud.resource.ResourceState.Event.InternalEnterMaintenance;
53+
import static com.cloud.resource.ResourceState.Event.UnableToMigrate;
54+
import static org.mockito.Matchers.any;
55+
import static org.mockito.Matchers.anyBoolean;
56+
import static org.mockito.Matchers.anyLong;
57+
import static org.mockito.Matchers.anyString;
58+
import static org.mockito.Matchers.eq;
59+
import static org.mockito.Mockito.times;
60+
import static org.mockito.Mockito.verify;
61+
import static org.mockito.Mockito.when;
62+
63+
@RunWith(PowerMockRunner.class)
64+
@PrepareForTest({ActionEventUtils.class, ResourceManagerImpl.class})
65+
public class ResourceManagerImplTest {
66+
67+
@Mock
68+
private CapacityDao capacityDao;
69+
@Mock
70+
private StorageManager storageManager;
71+
@Mock
72+
private HighAvailabilityManager haManager;
73+
@Mock
74+
private UserVmDetailsDao userVmDetailsDao;
75+
@Mock
76+
private AgentManager agentManager;
77+
@Mock
78+
private HostDao hostDao;
79+
@Mock
80+
private VMInstanceDao vmInstanceDao;
81+
82+
@Spy
83+
@InjectMocks
84+
private ResourceManagerImpl resourceManager = new ResourceManagerImpl();
85+
86+
@Mock
87+
private HostVO host;
88+
@Mock
89+
private VMInstanceVO vm1;
90+
@Mock
91+
private VMInstanceVO vm2;
92+
93+
@Mock
94+
private GetVncPortAnswer getVncPortAnswerVm1;
95+
@Mock
96+
private GetVncPortAnswer getVncPortAnswerVm2;
97+
@Mock
98+
private GetVncPortCommand getVncPortCommandVm1;
99+
@Mock
100+
private GetVncPortCommand getVncPortCommandVm2;
101+
102+
private static long hostId = 1L;
103+
104+
private static long vm1Id = 1L;
105+
private static String vm1InstanceName = "i-1-VM";
106+
private static long vm2Id = 2L;
107+
private static String vm2InstanceName = "i-2-VM";
108+
109+
private static String vm1VncAddress = "10.2.2.2";
110+
private static int vm1VncPort = 5900;
111+
private static String vm2VncAddress = "10.2.2.2";
112+
private static int vm2VncPort = 5901;
113+
114+
@Before
115+
public void setup() throws Exception {
116+
MockitoAnnotations.initMocks(this);
117+
when(host.getType()).thenReturn(Host.Type.Routing);
118+
when(host.getId()).thenReturn(hostId);
119+
when(host.getResourceState()).thenReturn(ResourceState.Enabled);
120+
when(host.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.VMware);
121+
when(hostDao.findById(hostId)).thenReturn(host);
122+
when(vm1.getId()).thenReturn(vm1Id);
123+
when(vm2.getId()).thenReturn(vm2Id);
124+
when(vm1.getInstanceName()).thenReturn(vm1InstanceName);
125+
when(vm2.getInstanceName()).thenReturn(vm2InstanceName);
126+
when(vmInstanceDao.listByHostId(hostId)).thenReturn(new ArrayList<>());
127+
when(vmInstanceDao.listVmsMigratingFromHost(hostId)).thenReturn(new ArrayList<>());
128+
when(vmInstanceDao.listNonMigratingVmsByHostEqualsLastHost(hostId)).thenReturn(new ArrayList<>());
129+
PowerMockito.mockStatic(ActionEventUtils.class);
130+
BDDMockito.given(ActionEventUtils.onCompletedActionEvent(anyLong(), anyLong(), anyString(), anyString(), anyString(), anyLong()))
131+
.willReturn(1L);
132+
when(getVncPortAnswerVm1.getAddress()).thenReturn(vm1VncAddress);
133+
when(getVncPortAnswerVm1.getPort()).thenReturn(vm1VncPort);
134+
when(getVncPortAnswerVm2.getAddress()).thenReturn(vm2VncAddress);
135+
when(getVncPortAnswerVm2.getPort()).thenReturn(vm2VncPort);
136+
PowerMockito.whenNew(GetVncPortCommand.class).withArguments(vm1Id, vm1InstanceName).thenReturn(getVncPortCommandVm1);
137+
PowerMockito.whenNew(GetVncPortCommand.class).withArguments(vm2Id, vm2InstanceName).thenReturn(getVncPortCommandVm2);
138+
when(agentManager.easySend(eq(hostId), eq(getVncPortCommandVm1))).thenReturn(getVncPortAnswerVm1);
139+
when(agentManager.easySend(eq(hostId), eq(getVncPortCommandVm2))).thenReturn(getVncPortAnswerVm2);
140+
}
141+
142+
@Test
143+
public void testCheckAndMaintainEnterMaintenanceMode() throws NoTransitionException {
144+
boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId);
145+
verify(resourceManager).isHostInMaintenance(host, new ArrayList<>(), new ArrayList<>(), new ArrayList<>());
146+
verify(resourceManager).setHostIntoMaintenance(host);
147+
verify(resourceManager).resourceStateTransitTo(eq(host), eq(InternalEnterMaintenance), anyLong());
148+
Assert.assertTrue(enterMaintenanceMode);
149+
}
150+
151+
@Test
152+
public void testCheckAndMaintainErrorInMaintenanceRunningVms() throws NoTransitionException {
153+
when(vmInstanceDao.listByHostId(hostId)).thenReturn(Arrays.asList(vm1, vm2));
154+
boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId);
155+
verify(resourceManager).isHostInMaintenance(host, Arrays.asList(vm1, vm2), new ArrayList<>(), new ArrayList<>());
156+
Assert.assertFalse(enterMaintenanceMode);
157+
}
158+
159+
@Test
160+
public void testCheckAndMaintainErrorInMaintenanceMigratingVms() throws NoTransitionException {
161+
when(vmInstanceDao.listVmsMigratingFromHost(hostId)).thenReturn(Arrays.asList(vm1, vm2));
162+
boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId);
163+
verify(resourceManager).isHostInMaintenance(host, new ArrayList<>(), Arrays.asList(vm1, vm2), new ArrayList<>());
164+
Assert.assertFalse(enterMaintenanceMode);
165+
}
166+
167+
@Test
168+
public void testCheckAndMaintainErrorInMaintenanceFailedMigrations() throws NoTransitionException {
169+
when(vmInstanceDao.listNonMigratingVmsByHostEqualsLastHost(hostId)).thenReturn(Arrays.asList(vm1, vm2));
170+
boolean enterMaintenanceMode = resourceManager.checkAndMaintain(hostId);
171+
verify(resourceManager).isHostInMaintenance(host, new ArrayList<>(), new ArrayList<>(), Arrays.asList(vm1, vm2));
172+
verify(resourceManager).setHostIntoErrorInMaintenance(host, Arrays.asList(vm1, vm2));
173+
verify(resourceManager).resourceStateTransitTo(eq(host), eq(UnableToMigrate), anyLong());
174+
Assert.assertFalse(enterMaintenanceMode);
175+
}
176+
177+
@Test
178+
public void testConfigureVncAccessForKVMHostFailedMigrations() {
179+
when(host.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM);
180+
List<VMInstanceVO> vms = Arrays.asList(vm1, vm2);
181+
resourceManager.configureVncAccessForKVMHostFailedMigrations(host, vms);
182+
verify(agentManager).pullAgentOutMaintenance(hostId);
183+
verify(resourceManager).setKVMVncAccess(hostId, vms);
184+
verify(agentManager, times(vms.size())).easySend(eq(hostId), any(GetVncPortCommand.class));
185+
verify(userVmDetailsDao).addDetail(eq(vm1Id), eq("kvm.vnc.address"), eq(vm1VncAddress), anyBoolean());
186+
verify(userVmDetailsDao).addDetail(eq(vm1Id), eq("kvm.vnc.port"), eq(String.valueOf(vm1VncPort)), anyBoolean());
187+
verify(userVmDetailsDao).addDetail(eq(vm2Id), eq("kvm.vnc.address"), eq(vm2VncAddress), anyBoolean());
188+
verify(userVmDetailsDao).addDetail(eq(vm2Id), eq("kvm.vnc.port"), eq(String.valueOf(vm2VncPort)), anyBoolean());
189+
verify(agentManager).pullAgentToMaintenance(hostId);
190+
}
191+
}

0 commit comments

Comments
 (0)