Merge branch '4.19'

Author: DaanHoogland

api/src/main/java/com/cloud/event/EventTypes.java

@@ -29,9 +29,9 @@
 import org.apache.cloudstack.api.response.ZoneResponse;
 import org.apache.cloudstack.config.Configuration;
 import org.apache.cloudstack.ha.HAConfig;
+import org.apache.cloudstack.quota.QuotaTariff;
 import org.apache.cloudstack.storage.object.Bucket;
 import org.apache.cloudstack.storage.object.ObjectStore;
-import org.apache.cloudstack.quota.QuotaTariff;
 import org.apache.cloudstack.usage.Usage;
 import org.apache.cloudstack.vm.schedule.VMSchedule;
 
@@ -1229,4 +1229,8 @@ public static Class getEntityClassForEvent(String eventName) {
     public static boolean isVpcEvent(String eventType) {
         return EventTypes.EVENT_VPC_CREATE.equals(eventType) || EventTypes.EVENT_VPC_DELETE.equals(eventType);
     }
+
+    public static void addEntityEventDetail(String event, Class<?> clazz) {
+        entityEventDetails.put(event, clazz);
+    }
 }

api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java

@@ -21,7 +21,7 @@
 import com.cloud.uservm.UserVm;
 import com.cloud.utils.component.Adapter;
 
-public interface KubernetesClusterHelper extends Adapter {
+public interface KubernetesServiceHelper extends Adapter {
 
     ControlledEntity findByUuid(String uuid);
     ControlledEntity findByVmId(long vmId);

api/src/main/java/com/cloud/network/NetworkService.java

@@ -20,6 +20,7 @@
 import java.util.Map;
 
 import com.cloud.dc.DataCenter;
+import org.apache.cloudstack.acl.ControlledEntity;
 import org.apache.cloudstack.api.command.admin.address.ReleasePodIpCmdByAdmin;
 import org.apache.cloudstack.api.command.admin.network.DedicateGuestVlanRangeCmd;
 import org.apache.cloudstack.api.command.admin.network.ListDedicatedGuestVlanRangesCmd;
@@ -102,6 +103,10 @@ IpAddress allocatePortableIP(Account ipOwner, int regionId, Long zoneId, Long ne
 
     Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapacityException, ConcurrentOperationException, ResourceAllocationException;
 
+    Network createGuestNetwork(long networkOfferingId, String name, String displayText, Account owner,
+           PhysicalNetwork physicalNetwork, long zoneId, ControlledEntity.ACLType aclType) throws
+            InsufficientCapacityException, ConcurrentOperationException, ResourceAllocationException;
+
     Pair<List<? extends Network>, Integer> searchForNetworks(ListNetworksCmd cmd);
 
     boolean deleteNetwork(long networkId, boolean forced);

api/src/main/java/com/cloud/vm/UserVmService.java

@@ -45,6 +45,7 @@
 import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.InsufficientCapacityException;
 import com.cloud.exception.ManagementServerException;
+import com.cloud.exception.OperationTimedoutException;
 import com.cloud.exception.ResourceAllocationException;
 import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.exception.StorageUnavailableException;
@@ -66,10 +67,7 @@ public interface UserVmService {
     /**
      * Destroys one virtual machine
      *
-     * @param userId
-     *            the id of the user performing the action
-     * @param vmId
-     *            the id of the virtual machine.
+     * @param cmd the API Command Object containg the parameters to use for this service action
      * @throws ConcurrentOperationException
      * @throws ResourceUnavailableException
      */
@@ -112,6 +110,8 @@ UserVm startVirtualMachine(StartVMCmd cmd) throws StorageUnavailableException, E
 
     UserVm rebootVirtualMachine(RebootVMCmd cmd) throws InsufficientCapacityException, ResourceUnavailableException, ResourceAllocationException;
 
+    void startVirtualMachine(UserVm vm) throws OperationTimedoutException, ResourceUnavailableException, InsufficientCapacityException;
+
     UserVm updateVirtualMachine(UpdateVMCmd cmd) throws ResourceUnavailableException, InsufficientCapacityException;
 
     /**
@@ -148,14 +148,6 @@ UserVm startVirtualMachine(StartVMCmd cmd) throws StorageUnavailableException, E
      * Creates a Basic Zone User VM in the database and returns the VM to the
      * caller.
      *
-     *
-     *
-     * @param sshKeyPair
-     *            - name of the ssh key pair used to login to the virtual
-     *            machine
-     * @param cpuSpeed
-     * @param memory
-     * @param cpuNumber
      * @param zone
      *            - availability zone for the virtual machine
      * @param serviceOffering
@@ -231,9 +223,6 @@ UserVm createBasicSecurityGroupVirtualMachine(DataCenter zone, ServiceOffering s
      * Creates a User VM in Advanced Zone (Security Group feature is enabled) in
      * the database and returns the VM to the caller.
      *
-     *
-     *
-     * @param type
      * @param zone
      *            - availability zone for the virtual machine
      * @param serviceOffering
@@ -309,14 +298,6 @@ UserVm createAdvancedSecurityGroupVirtualMachine(DataCenter zone, ServiceOfferin
      * Creates a User VM in Advanced Zone (Security Group feature is disabled)
      * in the database and returns the VM to the caller.
      *
-     *
-     *
-     * @param sshKeyPair
-     *            - name of the ssh key pair used to login to the virtual
-     *            machine
-     * @param cpuSpeed
-     * @param memory
-     * @param cpuNumber
      * @param zone
      *            - availability zone for the virtual machine
      * @param serviceOffering

api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java

@@ -17,7 +17,9 @@
 package org.apache.cloudstack.api;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.cloudstack.region.PortableIp;
 import org.apache.commons.collections.CollectionUtils;
@@ -81,15 +83,22 @@ public enum ApiCommandResourceType {
     ManagementServer(org.apache.cloudstack.management.ManagementServerHost.class),
     ObjectStore(org.apache.cloudstack.storage.object.ObjectStore.class),
     Bucket(org.apache.cloudstack.storage.object.Bucket.class),
-    QuotaTariff(org.apache.cloudstack.quota.QuotaTariff.class);
+    QuotaTariff(org.apache.cloudstack.quota.QuotaTariff.class),
+    KubernetesCluster(null),
+    KubernetesSupportedVersion(null);
 
     private final Class<?> clazz;
 
+    static final Map<ApiCommandResourceType, Class<?>> additionalClassMappings = new HashMap<>();
+
     private ApiCommandResourceType(Class<?> clazz) {
         this.clazz = clazz;
     }
 
     public Class<?> getAssociatedClass() {
+        if (this.clazz == null && additionalClassMappings.containsKey(this)) {
+            return additionalClassMappings.get(this);
+        }
         return this.clazz;
     }
 
@@ -119,4 +128,8 @@ public static ApiCommandResourceType fromString(String value) {
         }
         return null;
     }
+
+    public static void setClassMapping(ApiCommandResourceType type, Class<?> clazz) {
+        additionalClassMappings.put(type, clazz);
+    }
 }

api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java

@@ -54,7 +54,11 @@ public class CreateServiceOfferingCmd extends BaseCmd {
     @Parameter(name = ApiConstants.CPU_NUMBER, type = CommandType.INTEGER, required = false, description = "the CPU number of the service offering")
     private Integer cpuNumber;
 
-    @Parameter(name = ApiConstants.CPU_SPEED, type = CommandType.INTEGER, required = false, description = "the CPU speed of the service offering in MHz.")
+    @Parameter(name = ApiConstants.CPU_SPEED, type = CommandType.INTEGER, required = false, description = "For VMware and Xen based hypervisors this is the CPU speed of the service offering in MHz.\n" +
+            "For the KVM hypervisor," +
+            " the values of the parameters cpuSpeed and cpuNumber will be used to calculate the `shares` value. This value is used by the KVM hypervisor to calculate how much time" +
+            " the VM will have access to the host's CPU. The `shares` value does not have a unit, and its purpose is being a weight value for the host to compare between its guest" +
+            " VMs. For more information, see https://libvirt.org/formatdomain.html#cpu-tuning.")
     private Integer cpuSpeed;
 
     @Parameter(name = ApiConstants.DISPLAY_TEXT, type = CommandType.STRING, description = "The display text of the service offering, defaults to 'name'.")

api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java

@@ -97,8 +97,8 @@ public class ListVMsCmd extends BaseListRetrieveOnlyResourceCountCmd implements
                collectionType = CommandType.STRING,
                description = "comma separated list of vm details requested, "
                    + "value can be a list of [all, group, nics, stats, secgrp, tmpl, servoff, diskoff, backoff, iso, volume, min, affgrp]."
-                   + " If no parameter is passed in, the details will be defaulted to all. When return.vm.stats.on.vm.list is true, the default" +
-                       "details change to [group, nics, secgrp, tmpl, servoff, diskoff, backoff, iso, volume, min, affgrp], thus the stats will not be returned. ")
+                   + " When no parameters are passed, all the details are returned if list.vm.default.details.stats is true (default),"
+                   + " otherwise when list.vm.default.details.stats is false the API response will exclude the stats details.")
     private List<String> viewDetails;
 
     @Parameter(name = ApiConstants.TEMPLATE_ID, type = CommandType.UUID, entityType = TemplateResponse.class, description = "list vms by template")

api/src/main/java/org/apache/cloudstack/query/QueryService.java

@@ -125,9 +125,8 @@ public interface QueryService {
     static final ConfigKey<Boolean> SharePublicTemplatesWithOtherDomains = new ConfigKey<>("Advanced", Boolean.class, "share.public.templates.with.other.domains", "true",
             "If false, templates of this domain will not show up in the list templates of other domains.", true, ConfigKey.Scope.Domain);
 
-    ConfigKey<Boolean> ReturnVmStatsOnVmList = new ConfigKey<>("Advanced", Boolean.class, "return.vm.stats.on.vm.list", "true",
-            "If false, changes the listVirtualMachines default details to [group, nics, secgrp, tmpl, servoff, diskoff, backoff, iso, volume, min, affgrp], so that the VMs' stats" +
-                    " are not returned by default when listing VMs; only when the 'stats' or 'all' detail is informed.", true, ConfigKey.Scope.Global);
+    ConfigKey<Boolean> ReturnVmStatsOnVmList = new ConfigKey<>("Advanced", Boolean.class, "list.vm.default.details.stats", "true",
+            "Determines whether VM stats should be returned when details are not explicitly specified in listVirtualMachines API request. When false, details default to [group, nics, secgrp, tmpl, servoff, diskoff, backoff, iso, volume, min, affgrp]. When true, all details are returned including 'stats'.", true, ConfigKey.Scope.Global);
 
     ListResponse<UserResponse> searchForUsers(ListUsersCmd cmd) throws PermissionDeniedException;
 

core/src/main/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml

@@ -339,7 +339,7 @@
           class="org.apache.cloudstack.spring.lifecycle.registry.ExtensionRegistry">
     </bean>
 
-    <bean id="kubernetesClusterHelperRegistry"
+    <bean id="kubernetesServiceHelperRegistry"
           class="org.apache.cloudstack.spring.lifecycle.registry.ExtensionRegistry">
     </bean>
 

core/src/main/resources/META-INF/cloudstack/kubernetes/spring-core-lifecycle-kubernetes-context-inheritable.xml

@@ -25,8 +25,8 @@
 >
 
     <bean class="org.apache.cloudstack.spring.lifecycle.registry.RegistryLifecycle">
-        <property name="registry" ref="kubernetesClusterHelperRegistry" />
-        <property name="typeClass" value="com.cloud.kubernetes.cluster.KubernetesClusterHelper" />
+        <property name="registry" ref="kubernetesServiceHelperRegistry" />
+        <property name="typeClass" value="com.cloud.kubernetes.cluster.KubernetesServiceHelper" />
     </bean>
 
 </beans>

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java

@@ -1185,8 +1185,9 @@ public VolumeVO doInTransaction(TransactionStatus status) {
                     logger.error("Unable to destroy existing volume [{}] due to [{}].", volumeToString, e.getMessage());
                 }
                 // In case of VMware VM will continue to use the old root disk until expunged, so force expunge old root disk
-                if (vm.getHypervisorType() == HypervisorType.VMware) {
-                    logger.info("Trying to expunge volume [{}] from primary data storage.", volumeToString);
+                // For system VM we do not need volume entry in Destroy state
+                if (vm.getHypervisorType() == HypervisorType.VMware || vm.getType().isUsedBySystem()) {
+                    logger.info(String.format("Trying to expunge volume [%s] from primary data storage.", volumeToString));
                     AsyncCallFuture<VolumeApiResult> future = volService.expungeVolumeAsync(volFactory.getVolume(existingVolume.getId()));
                     try {
                         future.get();

engine/schema/src/main/resources/META-INF/db/schema-41900to41910-cleanup.sql

@@ -18,3 +18,7 @@
 --;
 -- Schema upgrade cleanup from 4.19.0.0 to 4.19.1.0
 --;
+
+-- List VMs response optimisation, don't sum during API handling
+UPDATE cloud.configuration set value='false' where name='vm.stats.increment.metrics';
+DELETE from cloud.configuration where name='vm.stats.increment.metrics.in.memory';

engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql

@@ -196,7 +196,7 @@ FROM
         LEFT JOIN `networks` ON ((`nics`.`network_id` = `networks`.`id`)))
         LEFT JOIN `vpc` ON (((`networks`.`vpc_id` = `vpc`.`id`)
         AND ISNULL(`vpc`.`removed`))))
-        LEFT JOIN `user_ip_address` ON ((`user_ip_address`.`vm_id` = `vm_instance`.`id`)))
+        LEFT JOIN `user_ip_address` FORCE INDEX(`fk_user_ip_address__vm_id`) ON ((`user_ip_address`.`vm_id` = `vm_instance`.`id`)))
         LEFT JOIN `user_vm_details` `ssh_details` ON (((`ssh_details`.`vm_id` = `vm_instance`.`id`)
         AND (`ssh_details`.`name` = 'SSH.KeyPairNames'))))
         LEFT JOIN `resource_tags` ON (((`resource_tags`.`resource_id` = `vm_instance`.`id`)

plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java

@@ -214,6 +214,7 @@ public void testGetUnmanagedInstanceDisks() {
 
         LibvirtDomainXMLParser parser = Mockito.mock(LibvirtDomainXMLParser.class);
         Mockito.when(parser.getDisks()).thenReturn(List.of(diskDef));
+        Mockito.doReturn(new Pair<String, String>(null, null)).when(convertInstanceCommandWrapper).getNfsStoragePoolHostAndPath(destinationPool);
 
         List<UnmanagedInstanceTO.Disk> unmanagedInstanceDisks = convertInstanceCommandWrapper.getUnmanagedInstanceDisks(disks, parser);
         Assert.assertEquals(1, unmanagedInstanceDisks.size());