Skip to content

Commit a22ab69

Browse files
committed
Set host into ErrorInMaintenance in case of failure trying to enter Maintenance mode
1 parent 08a8330 commit a22ab69

File tree

2 files changed

+79
-13
lines changed

2 files changed

+79
-13
lines changed

server/src/com/cloud/resource/ResourceManagerImpl.java

+70-12
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import javax.inject.Inject;
3131
import javax.naming.ConfigurationException;
3232

33+
import com.cloud.vm.dao.UserVmDetailsDao;
3334
import org.apache.commons.lang.ObjectUtils;
3435
import org.apache.log4j.Logger;
3536
import org.springframework.stereotype.Component;
@@ -54,6 +55,8 @@
5455
import com.cloud.agent.AgentManager;
5556
import com.cloud.agent.api.Answer;
5657
import com.cloud.agent.api.Command;
58+
import com.cloud.agent.api.GetVncPortCommand;
59+
import com.cloud.agent.api.GetVncPortAnswer;
5760
import com.cloud.agent.api.GetGPUStatsAnswer;
5861
import com.cloud.agent.api.GetGPUStatsCommand;
5962
import com.cloud.agent.api.GetHostStatsAnswer;
@@ -252,6 +255,8 @@ public void setDiscoverers(final List<? extends Discoverer> discoverers) {
252255
private ConfigurationManager _configMgr;
253256
@Inject
254257
private ClusterVSMMapDao _clusterVSMMapDao;
258+
@Inject
259+
private UserVmDetailsDao userVmDetailsDao;
255260

256261
private final long _nodeId = ManagementServerNode.getManagementServerId();
257262

@@ -1287,6 +1292,68 @@ public Host maintain(final PrepareForMaintenanceCmd cmd) {
12871292
}
12881293
}
12891294

1295+
/**
1296+
* Add VNC details as user VM details for each VM in 'vms' (KVM hosts only)
1297+
*/
1298+
private void setKVMVncAccess(long hostId, List<VMInstanceVO> vms) {
1299+
for (VMInstanceVO vm : vms) {
1300+
GetVncPortAnswer vmVncPortAnswer = (GetVncPortAnswer) _agentMgr.easySend(hostId, new GetVncPortCommand(vm.getId(), vm.getInstanceName()));
1301+
if (vmVncPortAnswer != null) {
1302+
userVmDetailsDao.addDetail(vm.getId(), "kvm.vnc.address", vmVncPortAnswer.getAddress(), true);
1303+
userVmDetailsDao.addDetail(vm.getId(), "kvm.vnc.port", String.valueOf(vmVncPortAnswer.getPort()), true);
1304+
}
1305+
}
1306+
}
1307+
1308+
/**
1309+
* Configure VNC access for host VMs which have failed migrating to another host while trying to enter Maintenance mode
1310+
*/
1311+
private void configureVncAccessForKVMHostFailedMigrations(HostVO host, List<VMInstanceVO> failedMigrations) {
1312+
if (host.getHypervisorType().equals(HypervisorType.KVM)) {
1313+
_agentMgr.pullAgentOutMaintenance(host.getId());
1314+
setKVMVncAccess(host.getId(), failedMigrations);
1315+
_agentMgr.pullAgentToMaintenance(host.getId());
1316+
}
1317+
}
1318+
1319+
/**
1320+
* Set host into ErrorInMaintenance state, as errors occurred during VM migrations. Do the following:
1321+
* - Cancel scheduled migrations for those which have already failed
1322+
* - Configure VNC access for VMs (KVM hosts only)
1323+
*/
1324+
private boolean setHostIntoErrorInMaintenance(HostVO host, List<VMInstanceVO> failedMigrations) throws NoTransitionException {
1325+
s_logger.debug("Unable to migrate " + failedMigrations.size() + " VM(s) from host " + host.getUuid());
1326+
_haMgr.cancelScheduledMigrations(host);
1327+
configureVncAccessForKVMHostFailedMigrations(host, failedMigrations);
1328+
resourceStateTransitTo(host, ResourceState.Event.UnableToMigrate, _nodeId);
1329+
return false;
1330+
}
1331+
1332+
/**
1333+
* Safely transit host into Maintenance mode
1334+
*/
1335+
private boolean setHostIntoMaintenance(HostVO host) throws NoTransitionException {
1336+
s_logger.debug("Host " + host.getUuid() + " entering in Maintenance");
1337+
resourceStateTransitTo(host, ResourceState.Event.InternalEnterMaintenance, _nodeId);
1338+
ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(),
1339+
EventVO.LEVEL_INFO, EventTypes.EVENT_MAINTENANCE_PREPARE,
1340+
"completed maintenance for host " + host.getId(), 0);
1341+
return true;
1342+
}
1343+
1344+
/**
1345+
* Return true if host goes into Maintenance mode, only when:
1346+
* - No Running, Migrating or Failed migrations (host_id = last_host_id) for the host
1347+
*/
1348+
private boolean isHostInMaintenance(HostVO host, List<VMInstanceVO> runningVms, List<VMInstanceVO> migratingVms, List<VMInstanceVO> failedMigrations) throws NoTransitionException {
1349+
if (CollectionUtils.isEmpty(runningVms) && CollectionUtils.isEmpty(migratingVms)) {
1350+
return CollectionUtils.isEmpty(failedMigrations) ?
1351+
setHostIntoMaintenance(host) :
1352+
setHostIntoErrorInMaintenance(host, failedMigrations);
1353+
}
1354+
return false;
1355+
}
1356+
12901357
@Override
12911358
public boolean checkAndMaintain(final long hostId) {
12921359
boolean hostInMaintenance = false;
@@ -1296,18 +1363,9 @@ public boolean checkAndMaintain(final long hostId) {
12961363
if (host.getType() != Host.Type.Storage) {
12971364
final List<VMInstanceVO> vos = _vmDao.listByHostId(hostId);
12981365
final List<VMInstanceVO> vosMigrating = _vmDao.listVmsMigratingFromHost(hostId);
1299-
final List<VMInstanceVO> failedMigratedVms = _vmDao.listNonMigratingVmsByHostEqualsLastHost(hostId);
1300-
if (vos.isEmpty() && vosMigrating.isEmpty()) {
1301-
if (!failedMigratedVms.isEmpty()) {
1302-
s_logger.debug("Unable to migrate " + failedMigratedVms.size() + " VM(s) from host " + host.getUuid());
1303-
resourceStateTransitTo(host, ResourceState.Event.UnableToMigrate, _nodeId);
1304-
} else {
1305-
s_logger.debug("Host " + host.getUuid() + " entering in Maintenance");
1306-
resourceStateTransitTo(host, ResourceState.Event.InternalEnterMaintenance, _nodeId);
1307-
hostInMaintenance = true;
1308-
ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_MAINTENANCE_PREPARE, "completed maintenance for host " + hostId, 0);
1309-
}
1310-
}
1366+
final List<VMInstanceVO> failedVmMigrations = _vmDao.listNonMigratingVmsByHostEqualsLastHost(hostId);
1367+
1368+
hostInMaintenance = isHostInMaintenance(host, vos, vosMigrating, failedVmMigrations);
13111369
}
13121370
} catch (final NoTransitionException e) {
13131371
s_logger.debug("Cannot transmit host " + host.getId() + "to Maintenance state", e);

server/src/com/cloud/servlet/ConsoleProxyServlet.java

+9-1
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import javax.servlet.http.HttpServletResponse;
3636
import javax.servlet.http.HttpSession;
3737

38+
import com.cloud.resource.ResourceState;
3839
import org.apache.commons.codec.binary.Base64;
3940
import org.apache.log4j.Logger;
4041
import org.springframework.stereotype.Component;
@@ -418,7 +419,14 @@ private String composeConsoleAccessUrl(String rootUrl, VirtualMachine vm, HostVO
418419
StringBuffer sb = new StringBuffer(rootUrl);
419420
String host = hostVo.getPrivateIpAddress();
420421

421-
Pair<String, Integer> portInfo = _ms.getVncPort(vm);
422+
Pair<String, Integer> portInfo;
423+
if (hostVo.getResourceState().equals(ResourceState.ErrorInMaintenance)) {
424+
UserVmDetailVO detailAddress = _userVmDetailsDao.findDetail(vm.getId(), "kvm.vnc.address");
425+
UserVmDetailVO detailPort = _userVmDetailsDao.findDetail(vm.getId(), "kvm.vnc.port");
426+
portInfo = new Pair<>(detailAddress.getValue(), Integer.valueOf(detailPort.getValue()));
427+
} else {
428+
portInfo = _ms.getVncPort(vm);
429+
}
422430
if (s_logger.isDebugEnabled())
423431
s_logger.debug("Port info " + portInfo.first());
424432

0 commit comments

Comments
 (0)