Merge branch '4.19'

Author: vishesh92

agent/src/main/java/com/cloud/agent/Agent.java

@@ -1127,6 +1127,12 @@ public void doTask(final Task task) throws TaskExecutionException {
                     logger.error("Error parsing task", e);
                 }
             } else if (task.getType() == Task.Type.DISCONNECT) {
+                try {
+                    // an issue has been found if reconnect immediately after disconnecting. please refer to https://github.com/apache/cloudstack/issues/8517
+                    // wait 5 seconds before reconnecting
+                    Thread.sleep(5000);
+                } catch (InterruptedException e) {
+                }
                 reconnect(task.getLink());
                 return;
             } else if (task.getType() == Task.Type.OTHER) {

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -751,7 +751,7 @@ public Property<Integer> getWorkers() {
     public static final Property<Integer> IOTHREADS = new Property<>("iothreads", 1);
 
     /**
-     * Enable verbose mode for virt-v2v Instance Conversion from Vmware to KVM
+     * Enable verbose mode for virt-v2v Instance Conversion from VMware to KVM
      * Data type: Boolean.<br>
      * Default value: <code>false</code>
      */

api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java

@@ -18,40 +18,39 @@
  */
 package com.cloud.agent.api.to;
 
+import java.io.Serializable;
+
 import com.cloud.agent.api.LogLevel;
 import com.cloud.hypervisor.Hypervisor;
 
-import java.io.Serializable;
-
 public class RemoteInstanceTO implements Serializable {
 
     private Hypervisor.HypervisorType hypervisorType;
-    private String hostName;
     private String instanceName;
 
-    // Vmware Remote Instances parameters
+    // VMware Remote Instances parameters (required for exporting OVA through ovftool)
     // TODO: cloud.agent.transport.Request#getCommands() cannot handle gsoc decode for polymorphic classes
     private String vcenterUsername;
     @LogLevel(LogLevel.Log4jLevel.Off)
     private String vcenterPassword;
     private String vcenterHost;
     private String datacenterName;
-    private String clusterName;
 
     public RemoteInstanceTO() {
     }
 
-    public RemoteInstanceTO(String hostName, String instanceName, String vcenterHost,
-                            String datacenterName, String clusterName,
-                            String vcenterUsername, String vcenterPassword) {
+    public RemoteInstanceTO(String instanceName) {
+        this.hypervisorType = Hypervisor.HypervisorType.VMware;
+        this.instanceName = instanceName;
+    }
+
+    public RemoteInstanceTO(String instanceName, String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName) {
         this.hypervisorType = Hypervisor.HypervisorType.VMware;
-        this.hostName = hostName;
         this.instanceName = instanceName;
         this.vcenterHost = vcenterHost;
-        this.datacenterName = datacenterName;
-        this.clusterName = clusterName;
         this.vcenterUsername = vcenterUsername;
         this.vcenterPassword = vcenterPassword;
+        this.datacenterName = datacenterName;
     }
 
     public Hypervisor.HypervisorType getHypervisorType() {
@@ -62,10 +61,6 @@ public String getInstanceName() {
         return this.instanceName;
     }
 
-    public String getHostName() {
-        return this.hostName;
-    }
-
     public String getVcenterUsername() {
         return vcenterUsername;
     }
@@ -81,8 +76,4 @@ public String getVcenterHost() {
     public String getDatacenterName() {
         return datacenterName;
     }
-
-    public String getClusterName() {
-        return clusterName;
-    }
 }

api/src/main/java/com/cloud/host/Host.java

@@ -54,6 +54,7 @@ public static String[] toStrings(Host.Type... types) {
     }
     public static final String HOST_UEFI_ENABLE = "host.uefi.enable";
     public static final String HOST_VOLUME_ENCRYPTION = "host.volume.encryption";
+    public static final String HOST_INSTANCE_CONVERSION = "host.instance.conversion";
 
     /**
      * @return name of the machine.

api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java

@@ -23,6 +23,7 @@
 import org.apache.cloudstack.framework.config.ConfigKey;
 
 import com.cloud.agent.api.Command;
+import com.cloud.agent.api.to.DataStoreTO;
 import com.cloud.agent.api.to.NicTO;
 import com.cloud.agent.api.to.VirtualMachineTO;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
@@ -101,21 +102,20 @@ boolean attachRestoredVolumeToVirtualMachine(long zoneId, String location, Backu
      * Will generate commands to migrate a vm to a pool. For now this will only work for stopped VMs on Vmware.
      *
      * @param vm the stopped vm to migrate
-     * @param destination the primary storage pool to migrate to
+     * @param volumeToPool the primary storage pools to migrate to
      * @return a list of commands to perform for a successful migration
      */
     List<Command> finalizeMigrate(VirtualMachine vm, Map<Volume, StoragePool> volumeToPool);
 
 
     /**
-     * Will perform a clone of a VM on an external host (if the guru can handle)
+     * Will return the hypervisor VM (clone VM for PowerOn VMs), performs a clone of a VM if required on an external host (if the guru can handle)
      * @param hostIp VM's source host IP
-     * @param vmName name of the source VM to clone from
+     * @param vmName name of the source VM (clone VM name if cloned)
      * @param params hypervisor specific additional parameters
-     * @return a reference to the cloned VM
+     * @return a reference to the hypervisor or cloned VM, and cloned flag
      */
-    UnmanagedInstanceTO cloneHypervisorVMOutOfBand(String hostIp, String vmName,
-                                                   Map<String, String> params);
+    Pair<UnmanagedInstanceTO, Boolean> getHypervisorVMOutOfBandAndCloneIfRequired(String hostIp, String vmName, Map<String, String> params);
 
     /**
      * Removes a VM created as a clone of a VM on an external host
@@ -124,6 +124,23 @@ UnmanagedInstanceTO cloneHypervisorVMOutOfBand(String hostIp, String vmName,
      * @param params hypervisor specific additional parameters
      * @return true if the operation succeeds, false if not
      */
-    boolean removeClonedHypervisorVMOutOfBand(String hostIp, String vmName,
-                                              Map<String, String> params);
+    boolean removeClonedHypervisorVMOutOfBand(String hostIp, String vmName, Map<String, String> params);
+
+    /**
+     * Create an OVA/OVF template of a VM on an external host (if the guru can handle)
+     * @param hostIp VM's source host IP
+     * @param vmName name of the source VM to create template from
+     * @param params hypervisor specific additional parameters
+     * @param templateLocation datastore to create the template file
+     * @return the created template dir/name
+     */
+    String createVMTemplateOutOfBand(String hostIp, String vmName, Map<String, String> params, DataStoreTO templateLocation, int threadsCountToExportOvf);
+
+    /**
+     * Removes the template on the location
+     * @param templateLocation datastore to remove the template file
+     * @param templateDir the template dir to remove from datastore
+     * @return true if the operation succeeds, false if not
+     */
+    boolean removeVMTemplateOutOfBand(DataStoreTO templateLocation, String templateDir);
 }

api/src/main/java/com/cloud/network/NetworkModel.java

@@ -317,6 +317,8 @@ public interface NetworkModel {
 
     void checkIp6Parameters(String startIPv6, String endIPv6, String ip6Gateway, String ip6Cidr) throws InvalidParameterValueException;
 
+    void checkIp6CidrSizeEqualTo64(String ip6Cidr) throws InvalidParameterValueException;
+
     void checkRequestedIpAddresses(long networkId, IpAddresses ips) throws InvalidParameterValueException;
 
     String getStartIpv6Address(long id);

api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java

@@ -17,17 +17,22 @@
 package com.cloud.network;
 
 import java.util.List;
+import java.util.Map;
 
 import org.apache.cloudstack.api.command.admin.router.UpgradeRouterCmd;
 import org.apache.cloudstack.api.command.admin.router.UpgradeRouterTemplateCmd;
 
+import com.cloud.deploy.DeploymentPlanner;
 import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.OperationTimedoutException;
 import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.network.router.VirtualRouter;
 import com.cloud.user.Account;
 import com.cloud.utils.Pair;
 import com.cloud.vm.Nic;
+import com.cloud.vm.VirtualMachine;
+import com.cloud.vm.VirtualMachineProfile;
 
 public interface VirtualNetworkApplianceService {
     /**
@@ -62,6 +67,10 @@ public interface VirtualNetworkApplianceService {
 
     VirtualRouter startRouter(long id) throws ResourceUnavailableException, InsufficientCapacityException, ConcurrentOperationException;
 
+    void startRouterForHA(VirtualMachine vm, Map<VirtualMachineProfile.Param, Object> params, DeploymentPlanner planner)
+            throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException,
+            OperationTimedoutException;
+
     VirtualRouter destroyRouter(long routerId, Account caller, Long callerUserId) throws ResourceUnavailableException, ConcurrentOperationException;
 
     VirtualRouter findRouter(long routerId);

api/src/main/java/com/cloud/network/guru/NetworkGuru.java

@@ -212,4 +212,7 @@ void reserve(NicProfile nic, Network network, VirtualMachineProfile vm, DeployDe
 
     boolean isMyTrafficType(TrafficType type);
 
+    default boolean isSlaacV6Only() {
+        return true;
+    }
 }

api/src/main/java/com/cloud/vm/NicProfile.java

@@ -62,6 +62,7 @@ public class NicProfile implements InternalIdentity, Serializable {
     String iPv4Dns1;
     String iPv4Dns2;
     String requestedIPv4;
+    boolean ipv4AllocationRaceCheck;
 
     // IPv6
     String iPv6Address;
@@ -405,6 +406,13 @@ public void setMtu(Integer mtu) {
         this.mtu = mtu;
     }
 
+    public boolean getIpv4AllocationRaceCheck() {
+        return this.ipv4AllocationRaceCheck;
+    }
+
+    public void setIpv4AllocationRaceCheck(boolean ipv4AllocationRaceCheck) {
+        this.ipv4AllocationRaceCheck = ipv4AllocationRaceCheck;
+    }
 
     //
     // OTHER METHODS

api/src/main/java/com/cloud/vm/UserVmService.java

@@ -42,6 +42,7 @@
 import org.apache.cloudstack.api.command.user.vmgroup.DeleteVMGroupCmd;
 
 import com.cloud.dc.DataCenter;
+import com.cloud.deploy.DeploymentPlanner;
 import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.InsufficientCapacityException;
 import com.cloud.exception.ManagementServerException;
@@ -112,6 +113,10 @@ UserVm startVirtualMachine(StartVMCmd cmd) throws StorageUnavailableException, E
 
     void startVirtualMachine(UserVm vm) throws OperationTimedoutException, ResourceUnavailableException, InsufficientCapacityException;
 
+    void startVirtualMachineForHA(VirtualMachine vm, Map<VirtualMachineProfile.Param, Object> params,
+          DeploymentPlanner planner) throws InsufficientCapacityException, ResourceUnavailableException,
+            ConcurrentOperationException, OperationTimedoutException;
+
     UserVm updateVirtualMachine(UpdateVMCmd cmd) throws ResourceUnavailableException, InsufficientCapacityException;
 
     /**

api/src/main/java/com/cloud/vm/VirtualMachineProfile.java

@@ -192,6 +192,10 @@ public boolean equals(Object obj) {
 
     Map<Param, Object> getParameters();
 
+    void setCpuOvercommitRatio(Float cpuOvercommitRatio);
+
+    void setMemoryOvercommitRatio(Float memoryOvercommitRatio);
+
     Float getCpuOvercommitRatio();
 
     Float getMemoryOvercommitRatio();

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -191,6 +191,7 @@ public class ApiConstants {
     public static final String FORCED = "forced";
     public static final String FORCED_DESTROY_LOCAL_STORAGE = "forcedestroylocalstorage";
     public static final String FORCE_DELETE_HOST = "forcedeletehost";
+    public static final String FORCE_MS_TO_IMPORT_VM_FILES = "forcemstoimportvmfiles";
     public static final String FORMAT = "format";
     public static final String FOR_VIRTUAL_NETWORK = "forvirtualnetwork";
     public static final String FOR_SYSTEM_VMS = "forsystemvms";
@@ -239,6 +240,7 @@ public class ApiConstants {
     public static final String NEXT_ACL_RULE_ID = "nextaclruleid";
     public static final String MOVE_ACL_CONSISTENCY_HASH = "aclconsistencyhash";
     public static final String IMAGE_PATH = "imagepath";
+    public static final String INSTANCE_CONVERSION_SUPPORTED = "instanceconversionsupported";
     public static final String INTERNAL_DNS1 = "internaldns1";
     public static final String INTERNAL_DNS2 = "internaldns2";
     public static final String INTERNET_PROTOCOL = "internetprotocol";
@@ -1135,6 +1137,8 @@ public class ApiConstants {
     public static final String WEBHOOK_ID = "webhookid";
     public static final String WEBHOOK_NAME = "webhookname";
 
+    public static final String NFS_MOUNT_OPTIONS = "nfsmountopts";
+
     /**
      * This enum specifies IO Drivers, each option controls specific policies on I/O.
      * Qemu guests support "threads" and "native" options Since 0.8.8 ; "io_uring" is supported Since 6.3.0 (QEMU 5.0).

api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java

@@ -37,6 +37,7 @@
 import org.apache.cloudstack.api.response.VmwareDatacenterResponse;
 import org.apache.cloudstack.api.response.ZoneResponse;
 import org.apache.cloudstack.vm.VmImportService;
+import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
 
@@ -116,40 +117,44 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd {
             description = "Temp Path on external host for disk image copy" )
     private String tmpPath;
 
-    // Import from Vmware to KVM migration parameters
+    // Import from VMware to KVM migration parameters
 
     @Parameter(name = ApiConstants.EXISTING_VCENTER_ID,
             type = CommandType.UUID,
             entityType = VmwareDatacenterResponse.class,
-            description = "(only for importing migrated VMs from Vmware to KVM) UUID of a linked existing vCenter")
+            description = "(only for importing VMs from VMware to KVM) UUID of a linked existing vCenter")
     private Long existingVcenterId;
 
     @Parameter(name = ApiConstants.HOST_IP,
             type = BaseCmd.CommandType.STRING,
-            description = "(only for importing migrated VMs from Vmware to KVM) VMware ESXi host IP/Name.")
+            description = "(only for importing VMs from VMware to KVM) VMware ESXi host IP/Name.")
     private String hostip;
 
     @Parameter(name = ApiConstants.VCENTER,
             type = CommandType.STRING,
-            description = "(only for importing migrated VMs from Vmware to KVM) The name/ip of vCenter. Make sure it is IP address or full qualified domain name for host running vCenter server.")
+            description = "(only for importing VMs from VMware to KVM) The name/ip of vCenter. Make sure it is IP address or full qualified domain name for host running vCenter server.")
     private String vcenter;
 
     @Parameter(name = ApiConstants.DATACENTER_NAME, type = CommandType.STRING,
-            description = "(only for importing migrated VMs from Vmware to KVM) Name of VMware datacenter.")
+            description = "(only for importing VMs from VMware to KVM) Name of VMware datacenter.")
     private String datacenterName;
 
     @Parameter(name = ApiConstants.CLUSTER_NAME, type = CommandType.STRING,
-            description = "(only for importing migrated VMs from Vmware to KVM) Name of VMware cluster.")
+            description = "(only for importing VMs from VMware to KVM) Name of VMware cluster.")
     private String clusterName;
 
     @Parameter(name = ApiConstants.CONVERT_INSTANCE_HOST_ID, type = CommandType.UUID, entityType = HostResponse.class,
-            description = "(only for importing migrated VMs from Vmware to KVM) optional - the host to perform the virt-v2v migration from VMware to KVM.")
+            description = "(only for importing VMs from VMware to KVM) optional - the host to perform the virt-v2v migration from VMware to KVM.")
     private Long convertInstanceHostId;
 
     @Parameter(name = ApiConstants.CONVERT_INSTANCE_STORAGE_POOL_ID, type = CommandType.UUID, entityType = StoragePoolResponse.class,
-            description = "(only for importing migrated VMs from Vmware to KVM) optional - the temporary storage pool to perform the virt-v2v migration from VMware to KVM.")
+            description = "(only for importing VMs from VMware to KVM) optional - the temporary storage pool to perform the virt-v2v migration from VMware to KVM.")
     private Long convertStoragePoolId;
 
+    @Parameter(name = ApiConstants.FORCE_MS_TO_IMPORT_VM_FILES, type = CommandType.BOOLEAN,
+            description = "(only for importing VMs from VMware to KVM) optional - if true, forces MS to import VM file(s) to temporary storage, else uses KVM Host if ovftool is available, falls back to MS if not.")
+    private Boolean forceMsToImportVmFiles;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -198,6 +203,10 @@ public Long getConvertStoragePoolId() {
         return convertStoragePoolId;
     }
 
+    public Boolean getForceMsToImportVmFiles() {
+        return BooleanUtils.toBooleanDefaultIfNull(forceMsToImportVmFiles, false);
+    }
+
     public String getHypervisor() {
         return hypervisor;
     }

api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/AssignToLoadBalancerRuleCmd.java

@@ -72,7 +72,7 @@ public class AssignToLoadBalancerRuleCmd extends BaseAsyncCmd {
 
     @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID_IP,
             type = CommandType.MAP,
-            description = "VM ID and IP map, vmidipmap[0].vmid=1 vmidipmap[0].ip=10.1.1.75",
+            description = "VM ID and IP map, vmidipmap[0].vmid=1 vmidipmap[0].vmip=10.1.1.75",
             since = "4.4")
     private Map vmIdIpMap;