Merge branch '4.20'

Author: DaanHoogland

api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java

@@ -27,6 +27,7 @@ public class RemoteInstanceTO implements Serializable {
 
     private Hypervisor.HypervisorType hypervisorType;
     private String instanceName;
+    private String instancePath;
 
     // VMware Remote Instances parameters (required for exporting OVA through ovftool)
     // TODO: cloud.agent.transport.Request#getCommands() cannot handle gsoc decode for polymorphic classes
@@ -44,9 +45,10 @@ public RemoteInstanceTO(String instanceName) {
         this.instanceName = instanceName;
     }
 
-    public RemoteInstanceTO(String instanceName, String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName) {
+    public RemoteInstanceTO(String instanceName, String instancePath, String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName) {
         this.hypervisorType = Hypervisor.HypervisorType.VMware;
         this.instanceName = instanceName;
+        this.instancePath = instancePath;
         this.vcenterHost = vcenterHost;
         this.vcenterUsername = vcenterUsername;
         this.vcenterPassword = vcenterPassword;
@@ -61,6 +63,10 @@ public String getInstanceName() {
         return this.instanceName;
     }
 
+    public String getInstancePath() {
+        return this.instancePath;
+    }
+
     public String getVcenterUsername() {
         return vcenterUsername;
     }

api/src/main/java/com/cloud/storage/VolumeApiService.java

@@ -22,7 +22,12 @@
 import java.util.List;
 import java.util.Map;
 
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.offering.DiskOffering;
+import com.cloud.user.Account;
 import com.cloud.utils.Pair;
+import com.cloud.utils.fsm.NoTransitionException;
+
 import org.apache.cloudstack.api.command.user.volume.AssignVolumeCmd;
 import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd;
 import org.apache.cloudstack.api.command.user.volume.ChangeOfferingForVolumeCmd;
@@ -37,21 +42,17 @@
 import org.apache.cloudstack.api.response.GetUploadParamsResponse;
 import org.apache.cloudstack.framework.config.ConfigKey;
 
-import com.cloud.exception.ResourceAllocationException;
-import com.cloud.user.Account;
-import com.cloud.utils.fsm.NoTransitionException;
-
 public interface VolumeApiService {
 
-    ConfigKey<Long> ConcurrentMigrationsThresholdPerDatastore = new ConfigKey<Long>("Advanced"
+    ConfigKey<Long> ConcurrentMigrationsThresholdPerDatastore = new ConfigKey<>("Advanced"
             , Long.class
             , "concurrent.migrations.per.target.datastore"
             , "0"
             , "Limits number of migrations that can be handled per datastore concurrently; default is 0 - unlimited"
             , true // not sure if this is to be dynamic
             , ConfigKey.Scope.Global);
 
-    ConfigKey<Boolean> UseHttpsToUpload = new ConfigKey<Boolean>("Advanced",
+    ConfigKey<Boolean> UseHttpsToUpload = new ConfigKey<>("Advanced",
             Boolean.class,
             "use.https.to.upload",
             "true",
@@ -85,7 +86,7 @@ public interface VolumeApiService {
      * @param cmd
      *            the API command wrapping the criteria
      * @return the volume object
-     * @throws ResourceAllocationException
+     * @throws ResourceAllocationException no capacity to allocate the new volume size
      */
     Volume resizeVolume(ResizeVolumeCmd cmd) throws ResourceAllocationException;
 
@@ -139,13 +140,13 @@ Volume updateVolume(long volumeId, String path, String state, Long storageId,
     Snapshot allocSnapshotForVm(Long vmId, Long volumeId, String snapshotName) throws ResourceAllocationException;
 
     /**
-     *  Checks if the target storage supports the disk offering.
+     *  Checks if the storage pool supports the disk offering tags.
      *  This validation is consistent with the mechanism used to select a storage pool to deploy a volume when a virtual machine is deployed or when a data disk is allocated.
      *
      *  The scenarios when this method returns true or false is presented in the following table.
      *   <table border="1">
      *      <tr>
-     *          <th>#</th><th>Disk offering tags</th><th>Storage tags</th><th>Does the storage support the disk offering?</th>
+     *          <th>#</th><th>Disk offering diskOfferingTags</th><th>Storage diskOfferingTags</th><th>Does the storage support the disk offering?</th>
      *      </tr>
      *      <body>
      *      <tr>
@@ -169,7 +170,8 @@ Volume updateVolume(long volumeId, String path, String state, Long storageId,
      *      </body>
      *   </table>
      */
-    boolean doesTargetStorageSupportDiskOffering(StoragePool destPool, String diskOfferingTags);
+    boolean doesStoragePoolSupportDiskOffering(StoragePool destPool, DiskOffering diskOffering);
+    boolean doesStoragePoolSupportDiskOfferingTags(StoragePool destPool, String diskOfferingTags);
 
     Volume destroyVolume(long volumeId, Account caller, boolean expunge, boolean forceExpunge);
 

api/src/main/java/org/apache/cloudstack/api/command/admin/offering/IsAccountAllowedToCreateOfferingsWithTagsCmd.java

@@ -29,7 +29,7 @@
         responseObject = IsAccountAllowedToCreateOfferingsWithTagsResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
 public class IsAccountAllowedToCreateOfferingsWithTagsCmd extends BaseCmd {
 
-    @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "Account UUID")
+    @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "Account UUID", required = true)
     private Long id;
 
     @Override

api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java

@@ -33,6 +33,8 @@ public enum PowerState {
 
     private String internalCSName;
 
+    private String path;
+
     private PowerState powerState;
 
     private PowerState cloneSourcePowerState;
@@ -75,6 +77,14 @@ public void setInternalCSName(String internalCSName) {
         this.internalCSName = internalCSName;
     }
 
+    public String getPath() {
+        return path;
+    }
+
+    public void setPath(String path) {
+        this.path = path;
+    }
+
     public PowerState getPowerState() {
         return powerState;
     }

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -707,7 +707,7 @@ protected void advanceExpunge(VMInstanceVO vm) throws ResourceUnavailableExcepti
 
     private void handleUnsuccessfulExpungeOperation(List<Command> finalizeExpungeCommands, List<Command> nicExpungeCommands,
                                                     VMInstanceVO vm, Long hostId) throws OperationTimedoutException, AgentUnavailableException {
-        if (CollectionUtils.isNotEmpty(finalizeExpungeCommands) || CollectionUtils.isNotEmpty(nicExpungeCommands) && (hostId != null)) {
+        if ((CollectionUtils.isNotEmpty(finalizeExpungeCommands) || CollectionUtils.isNotEmpty(nicExpungeCommands)) && hostId != null) {
             final Commands cmds = new Commands(Command.OnError.Stop);
             addAllExpungeCommandsFromList(finalizeExpungeCommands, cmds, vm);
             addAllExpungeCommandsFromList(nicExpungeCommands, cmds, vm);

engine/schema/src/main/java/com/cloud/usage/dao/UsageNetworksDaoImpl.java

@@ -16,7 +16,6 @@
 // under the License.
 package com.cloud.usage.dao;
 
-import com.cloud.network.Network;
 import com.cloud.usage.UsageNetworksVO;
 import com.cloud.utils.DateUtil;
 import com.cloud.utils.db.GenericDaoBase;
@@ -70,11 +69,10 @@ public void remove(long networkId, Date removed) {
             SearchCriteria<UsageNetworksVO> sc = this.createSearchCriteria();
             sc.addAnd("networkId", SearchCriteria.Op.EQ, networkId);
             sc.addAnd("removed", SearchCriteria.Op.NULL);
-            UsageNetworksVO vo = findOneBy(sc);
-            if (vo != null) {
-                vo.setRemoved(removed);
-                vo.setState(Network.State.Destroy.name());
-                update(vo.getId(), vo);
+            List<UsageNetworksVO> usageNetworksVOs = listBy(sc);
+            for (UsageNetworksVO entry : usageNetworksVOs) {
+                entry.setRemoved(removed);
+                update(entry.getId(), entry);
             }
         } catch (final Exception e) {
             txn.rollback();

engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDaoImpl.java

@@ -16,7 +16,6 @@
 // under the License.
 package com.cloud.usage.dao;
 
-import com.cloud.network.vpc.Vpc;
 import com.cloud.usage.UsageVpcVO;
 import com.cloud.utils.DateUtil;
 import com.cloud.utils.db.GenericDaoBase;
@@ -64,11 +63,10 @@ public void remove(long vpcId, Date removed) {
             SearchCriteria<UsageVpcVO> sc = this.createSearchCriteria();
             sc.addAnd("vpcId", SearchCriteria.Op.EQ, vpcId);
             sc.addAnd("removed", SearchCriteria.Op.NULL);
-            UsageVpcVO vo = findOneBy(sc);
-            if (vo != null) {
-                vo.setRemoved(removed);
-                vo.setState(Vpc.State.Inactive.name());
-                update(vo.getId(), vo);
+            List<UsageVpcVO> usageVpcVOs = listBy(sc);
+            for (UsageVpcVO entry : usageVpcVOs) {
+                entry.setRemoved(removed);
+                update(entry.getId(), entry);
             }
         } catch (final Exception e) {
             txn.rollback();

plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java

@@ -45,6 +45,8 @@
 import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMap;
 import com.cloud.dc.dao.VmwareDatacenterDao;
 import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao;
+import com.cloud.storage.dao.VolumeDao;
+import com.cloud.user.User;
 import com.cloud.utils.Pair;
 import com.cloud.utils.component.AdapterBase;
 import com.cloud.utils.exception.CloudRuntimeException;
@@ -100,6 +102,8 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider,
     private AgentManager agentMgr;
     @Inject
     private VirtualMachineManager virtualMachineManager;
+    @Inject
+    private VolumeDao volumeDao;
 
     protected VeeamClient getClient(final Long zoneId) {
         try {
@@ -340,6 +344,59 @@ public List<Backup.RestorePoint> listRestorePoints(VirtualMachine vm) {
         return getClient(vm.getDataCenterId()).listRestorePoints(backupName, vm.getInstanceName());
     }
 
+    @Override
+    public void syncBackups(VirtualMachine vm, Backup.Metric metric) {
+        List<Backup.RestorePoint> restorePoints = listRestorePoints(vm);
+        if (CollectionUtils.isEmpty(restorePoints)) {
+            logger.debug("Can't find any restore point to VM: {}", vm);
+            return;
+        }
+        Transaction.execute(new TransactionCallbackNoReturn() {
+            @Override
+            public void doInTransactionWithoutResult(TransactionStatus status) {
+                final List<Backup> backupsInDb = backupDao.listByVmId(null, vm.getId());
+                final List<Long> removeList = backupsInDb.stream().map(InternalIdentity::getId).collect(Collectors.toList());
+                for (final Backup.RestorePoint restorePoint : restorePoints) {
+                    if (!(restorePoint.getId() == null || restorePoint.getType() == null || restorePoint.getCreated() == null)) {
+                        Backup existingBackupEntry = checkAndUpdateIfBackupEntryExistsForRestorePoint(backupsInDb, restorePoint, metric);
+                        if (existingBackupEntry != null) {
+                            removeList.remove(existingBackupEntry.getId());
+                            continue;
+                        }
+
+                        BackupVO backup = new BackupVO();
+                        backup.setVmId(vm.getId());
+                        backup.setExternalId(restorePoint.getId());
+                        backup.setType(restorePoint.getType());
+                        backup.setDate(restorePoint.getCreated());
+                        backup.setStatus(Backup.Status.BackedUp);
+                        if (metric != null) {
+                            backup.setSize(metric.getBackupSize());
+                            backup.setProtectedSize(metric.getDataSize());
+                        }
+                        backup.setBackupOfferingId(vm.getBackupOfferingId());
+                        backup.setAccountId(vm.getAccountId());
+                        backup.setDomainId(vm.getDomainId());
+                        backup.setZoneId(vm.getDataCenterId());
+                        backup.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId())));
+
+                        logger.debug("Creating a new entry in backups: [id: {}, uuid: {}, name: {}, vm_id: {}, external_id: {}, type: {}, date: {}, backup_offering_id: {}, account_id: {}, "
+                                + "domain_id: {}, zone_id: {}].", backup.getId(), backup.getUuid(), backup.getName(), backup.getVmId(), backup.getExternalId(), backup.getType(), backup.getDate(), backup.getBackupOfferingId(), backup.getAccountId(), backup.getDomainId(), backup.getZoneId());
+                        backupDao.persist(backup);
+
+                        ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_VM_BACKUP_CREATE,
+                                String.format("Created backup %s for VM ID: %s", backup.getUuid(), vm.getUuid()),
+                                vm.getId(), ApiCommandResourceType.VirtualMachine.toString(),0);
+                    }
+                }
+                for (final Long backupIdToRemove : removeList) {
+                    logger.warn(String.format("Removing backup with ID: [%s].", backupIdToRemove));
+                    backupDao.remove(backupIdToRemove);
+                }
+            }
+        });
+    }
+
     @Override
     public String getConfigComponentName() {
         return BackupService.class.getSimpleName();

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java

@@ -70,7 +70,6 @@ public Answer execute(ConvertInstanceCommand cmd, LibvirtComputingResource serve
         Hypervisor.HypervisorType sourceHypervisorType = sourceInstance.getHypervisorType();
         String sourceInstanceName = sourceInstance.getInstanceName();
         Hypervisor.HypervisorType destinationHypervisorType = cmd.getDestinationHypervisorType();
-        List<String> destinationStoragePools = cmd.getDestinationStoragePools();
         DataStoreTO conversionTemporaryLocation = cmd.getConversionTemporaryLocation();
         long timeout = (long) cmd.getWait() * 1000;
 
@@ -193,9 +192,15 @@ private String getExportOVAUrlFromRemoteInstance(RemoteInstanceTO vmwareInstance
         String password = vmwareInstance.getVcenterPassword();
         String datacenter = vmwareInstance.getDatacenterName();
         String vm = vmwareInstance.getInstanceName();
+        String path = vmwareInstance.getInstancePath();
 
         String encodedUsername = encodeUsername(username);
         String encodedPassword = encodeUsername(password);
+        if (StringUtils.isNotBlank(path)) {
+            logger.info("VM path: {}", path);
+            return String.format("vi://%s:%s@%s/%s/%s/%s",
+                    encodedUsername, encodedPassword, vcenter, datacenter, path, vm);
+        }
         return String.format("vi://%s:%s@%s/%s/vm/%s",
                 encodedUsername, encodedPassword, vcenter, datacenter, vm);
     }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java

@@ -2472,8 +2472,15 @@ public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd)
 
             template = storagePoolMgr.createPhysicalDiskFromDirectDownloadTemplate(tempFilePath, destTemplatePath, destPool, cmd.getFormat(), cmd.getWaitInMillSeconds());
 
-            String templatePath = template.getPath();
-            if (templatePath != null) {
+            String templatePath = null;
+            if (template != null) {
+                templatePath = template.getPath();
+            }
+            if (StringUtils.isEmpty(templatePath)) {
+                logger.warn("Skipped validation whether downloaded file is QCOW2 for template {}, due to downloaded template path is empty", template.getName());
+            } else if (!new File(templatePath).exists()) {
+                logger.warn("Skipped validation whether downloaded file is QCOW2 for template {}, due to downloaded template path is not valid: {}", template.getName(), templatePath);
+            } else {
                 try {
                     Qcow2Inspector.validateQcow2File(templatePath);
                 } catch (RuntimeException e) {

plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java

@@ -58,9 +58,8 @@ public class KVMHostInfo {
     private long reservedMemory;
     private long overCommitMemory;
     private List<String> capabilities = new ArrayList<>();
-
-    private static String cpuInfoFreqFileName = "/sys/devices/system/cpu/cpu0/cpufreq/base_frequency";
     private static String cpuArchCommand = "/usr/bin/arch";
+    private static List<String> cpuInfoFreqFileNames = List.of("/sys/devices/system/cpu/cpu0/cpufreq/base_frequency","/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq");
 
     public KVMHostInfo(long reservedMemory, long overCommitMemory, long manualSpeed, int reservedCpus) {
         this.cpuSpeed = manualSpeed;
@@ -138,32 +137,44 @@ protected static long getCpuSpeed(final String cpabilities, final NodeInfo nodeI
     }
 
     private static long getCpuSpeedFromCommandLscpu() {
+        long speed = 0L;
+        LOGGER.info("Fetching CPU speed from command \"lscpu\".");
         try {
-            LOGGER.info("Fetching CPU speed from command \"lscpu\".");
             String command = "lscpu | grep -i 'Model name' | head -n 1 | egrep -o '[[:digit:]].[[:digit:]]+GHz' | sed 's/GHz//g'";
             if(isHostS390x()) {
                 command = "lscpu | grep 'CPU dynamic MHz' | cut -d ':' -f 2 | tr -d ' ' | awk '{printf \"%.1f\\n\", $1 / 1000}'";
             }
             String result = Script.runSimpleBashScript(command);
-            long speed = (long) (Float.parseFloat(result) * 1000);
+            speed = (long) (Float.parseFloat(result) * 1000);
             LOGGER.info(String.format("Command [%s] resulted in the value [%s] for CPU speed.", command, speed));
             return speed;
         } catch (NullPointerException | NumberFormatException e) {
             LOGGER.error(String.format("Unable to retrieve the CPU speed from lscpu."), e);
-            return 0L;
         }
+        try {
+            String command = "lscpu | grep -i 'CPU max MHz' | head -n 1 | sed 's/^.*: //' | xargs";
+            String result = Script.runSimpleBashScript(command);
+            speed = (long) (Float.parseFloat(result));
+            LOGGER.info(String.format("Command [%s] resulted in the value [%s] for CPU speed.", command, speed));
+            return speed;
+        } catch (NullPointerException | NumberFormatException e) {
+            LOGGER.error(String.format("Unable to retrieve the CPU speed from lscpu."), e);
+        }
+        return speed;
     }
 
     private static long getCpuSpeedFromFile() {
-        LOGGER.info(String.format("Fetching CPU speed from file [%s].", cpuInfoFreqFileName));
-        try (Reader reader = new FileReader(cpuInfoFreqFileName)) {
-            Long cpuInfoFreq = Long.parseLong(IOUtils.toString(reader).trim());
-            LOGGER.info(String.format("Retrieved value [%s] from file [%s]. This corresponds to a CPU speed of [%s] MHz.", cpuInfoFreq, cpuInfoFreqFileName, cpuInfoFreq / 1000));
-            return cpuInfoFreq / 1000;
-        } catch (IOException | NumberFormatException e) {
-            LOGGER.error(String.format("Unable to retrieve the CPU speed from file [%s]", cpuInfoFreqFileName), e);
-            return 0L;
+        for (final String cpuInfoFreqFileName:  cpuInfoFreqFileNames) {
+            LOGGER.info(String.format("Fetching CPU speed from file [%s].", cpuInfoFreqFileName));
+            try (Reader reader = new FileReader(cpuInfoFreqFileName)) {
+                Long cpuInfoFreq = Long.parseLong(IOUtils.toString(reader).trim());
+                LOGGER.info(String.format("Retrieved value [%s] from file [%s]. This corresponds to a CPU speed of [%s] MHz.", cpuInfoFreq, cpuInfoFreqFileName, cpuInfoFreq / 1000));
+                return cpuInfoFreq / 1000;
+            } catch (IOException | NumberFormatException e) {
+                LOGGER.error(String.format("Unable to retrieve the CPU speed from file [%s]", cpuInfoFreqFileName), e);
+            }
         }
+        return 0L;
     }
 
     protected static long getCpuSpeedFromHostCapabilities(final String capabilities) {