New feature: Implicit host tags (apache#8929)

* Merge two HostTagVO and HostTagDaoImpl

* Implicit host tags

* PR8929: add since

* Update variable names

* Update 8929: add unit test in LibvirtComputingResourceTest

* Update 8929: add explicithosttags in response

* Update 8929 UI: Update explicit host tags

* Update 8929: remove host tags and change labels on UI

* Update 8929: update host_view to use explicit_host_tags.is_tag_a_rule

* Update: ui polish for host tags

* Update 8929: fix UI error if no host tags

Author: weizhouapache

agent/conf/agent.properties

@@ -430,3 +430,6 @@ iscsi.session.cleanup.enabled=false
 # If set to "true", the agent will register for libvirt domain events, allowing for immediate updates on crashed or
 # unexpectedly stopped. Experimental, requires agent restart.
 # libvirt.events.enabled=false
+
+# Implicit host tags managed by agent.properties
+# host.tags=

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -803,6 +803,13 @@ public Property<Integer> getWorkers() {
      */
     public static final Property<String> KEYSTORE_PASSPHRASE = new Property<>(KeyStoreUtils.KS_PASSPHRASE_PROPERTY, null, String.class);
 
+    /**
+     * Implicit host tags
+     * Data type: String.<br>
+     * Default value: <code>null</code>
+     */
+    public static final Property<String> HOST_TAGS = new Property<>("host.tags", null, String.class);
+
     public static class Property <T>{
         private String name;
         private T defaultValue;

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -265,6 +265,7 @@ public class ApiConstants {
     public static final String IS_EDGE = "isedge";
     public static final String IS_EXTRACTABLE = "isextractable";
     public static final String IS_FEATURED = "isfeatured";
+    public static final String IS_IMPLICIT = "isimplicit";
     public static final String IS_PORTABLE = "isportable";
     public static final String IS_PUBLIC = "ispublic";
     public static final String IS_PERSISTENT = "ispersistent";

api/src/main/java/org/apache/cloudstack/api/response/HostForMigrationResponse.java

@@ -208,6 +208,14 @@ public class HostForMigrationResponse extends BaseResponse {
     @Param(description = "comma-separated list of tags for the host")
     private String hostTags;
 
+    @SerializedName("explicithosttags")
+    @Param(description = "comma-separated list of explicit host tags for the host", since = "4.20.0")
+    private String explicitHostTags;
+
+    @SerializedName("implicithosttags")
+    @Param(description = "comma-separated list of implicit host tags for the host", since = "4.20.0")
+    private String implicitHostTags;
+
     @SerializedName("hasenoughcapacity")
     @Param(description = "true if this host has enough CPU and RAM capacity to migrate a VM to it, false otherwise")
     private Boolean hasEnoughCapacity;
@@ -414,6 +422,14 @@ public void setHostTags(String hostTags) {
         this.hostTags = hostTags;
     }
 
+    public void setExplicitHostTags(String explicitHostTags) {
+        this.explicitHostTags = explicitHostTags;
+    }
+
+    public void setImplicitHostTags(String implicitHostTags) {
+        this.implicitHostTags = implicitHostTags;
+    }
+
     public void setHasEnoughCapacity(Boolean hasEnoughCapacity) {
         this.hasEnoughCapacity = hasEnoughCapacity;
     }

api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java

@@ -221,6 +221,14 @@ public class HostResponse extends BaseResponseWithAnnotations {
     @Param(description = "comma-separated list of tags for the host")
     private String hostTags;
 
+    @SerializedName("explicithosttags")
+    @Param(description = "comma-separated list of explicit host tags for the host", since = "4.20.0")
+    private String explicitHostTags;
+
+    @SerializedName("implicithosttags")
+    @Param(description = "comma-separated list of implicit host tags for the host", since = "4.20.0")
+    private String implicitHostTags;
+
     @SerializedName(ApiConstants.IS_TAG_A_RULE)
     @Param(description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE)
     private Boolean isTagARule;
@@ -458,6 +466,22 @@ public void setHostTags(String hostTags) {
         this.hostTags = hostTags;
     }
 
+    public String getExplicitHostTags() {
+        return explicitHostTags;
+    }
+
+    public void setExplicitHostTags(String explicitHostTags) {
+        this.explicitHostTags = explicitHostTags;
+    }
+
+    public String getImplicitHostTags() {
+        return implicitHostTags;
+    }
+
+    public void setImplicitHostTags(String implicitHostTags) {
+        this.implicitHostTags = implicitHostTags;
+    }
+
     public void setHasEnoughCapacity(Boolean hasEnoughCapacity) {
         this.hasEnoughCapacity = hasEnoughCapacity;
     }

api/src/main/java/org/apache/cloudstack/api/response/HostTagResponse.java

@@ -19,6 +19,7 @@
 import com.google.gson.annotations.SerializedName;
 import com.cloud.serializer.Param;
 
+import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.BaseResponse;
 
 public class HostTagResponse extends BaseResponse {
@@ -34,6 +35,10 @@ public class HostTagResponse extends BaseResponse {
     @Param(description = "the name of the host tag")
     private String name;
 
+    @SerializedName(ApiConstants.IS_IMPLICIT)
+    @Param(description = "true if the host tag is implicit", since = "4.20.0")
+    private boolean isImplicit;
+
     public String getId() {
         return id;
     }
@@ -57,4 +62,12 @@ public String getName() {
     public void setName(String name) {
         this.name = name;
     }
+
+    public boolean isImplicit() {
+        return isImplicit;
+    }
+
+    public void setImplicit(boolean implicit) {
+        isImplicit = implicit;
+    }
 }

core/src/main/java/com/cloud/agent/api/StartupRoutingCommand.java

@@ -174,6 +174,10 @@ public void setHostTags(String hostTag) {
         this.hostTags.add(hostTag);
     }
 
+    public void setHostTags(List<String> hostTags) {
+        this.hostTags = hostTags;
+    }
+
     public  HashMap<String, HashMap<String, VgpuTypesInfo>> getGpuGroupDetails() {
         return groupDetails;
     }

engine/schema/src/main/java/com/cloud/host/HostTagVO.java

@@ -40,6 +40,9 @@ public class HostTagVO implements InternalIdentity {
     @Column(name = "tag")
     private String tag;
 
+    @Column(name = "is_implicit")
+    private boolean isImplicit = false;
+
     @Column(name = "is_tag_a_rule")
     private boolean isTagARule;
 
@@ -74,6 +77,13 @@ public boolean getIsTagARule() {
         return isTagARule;
     }
 
+    public void setIsImplicit(boolean isImplicit) {
+        this.isImplicit = isImplicit;
+    }
+
+    public boolean getIsImplicit() {
+        return isImplicit;
+    }
 
     @Override
     public long getId() {

engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java

@@ -20,6 +20,7 @@
 
 import com.cloud.host.HostTagVO;
 import com.cloud.utils.db.GenericDao;
+import org.apache.cloudstack.api.response.HostTagResponse;
 import org.apache.cloudstack.framework.config.ConfigKey;
 
 public interface HostTagsDao extends GenericDao<HostTagVO, Long> {
@@ -35,6 +36,13 @@ public interface HostTagsDao extends GenericDao<HostTagVO, Long> {
 
     void deleteTags(long hostId);
 
+    boolean updateImplicitTags(long hostId, List<String> hostTags);
+
+    List<HostTagVO> getExplicitHostTags(long hostId);
+
     List<HostTagVO> findHostRuleTags();
 
+    HostTagResponse newHostTagResponse(HostTagVO hostTag);
+
+    List<HostTagVO> searchByIds(Long... hostTagIds);
 }

engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java

@@ -16,10 +16,14 @@
 // under the License.
 package com.cloud.host.dao;
 
+import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.cloudstack.api.response.HostTagResponse;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.Configurable;
+import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Component;
 
 import com.cloud.host.HostTagVO;
@@ -30,14 +34,23 @@
 import com.cloud.utils.db.TransactionLegacy;
 import com.cloud.utils.db.SearchCriteria.Func;
 
+import javax.inject.Inject;
+
 @Component
 public class HostTagsDaoImpl extends GenericDaoBase<HostTagVO, Long> implements HostTagsDao, Configurable {
     protected final SearchBuilder<HostTagVO> HostSearch;
     protected final GenericSearchBuilder<HostTagVO, String> DistinctImplictTagsSearch;
+    private final SearchBuilder<HostTagVO> stSearch;
+    private final SearchBuilder<HostTagVO> tagIdsearch;
+    private final SearchBuilder<HostTagVO> ImplicitTagsSearch;
+
+    @Inject
+    private ConfigurationDao _configDao;
 
     public HostTagsDaoImpl() {
         HostSearch = createSearchBuilder();
         HostSearch.and("hostId", HostSearch.entity().getHostId(), SearchCriteria.Op.EQ);
+        HostSearch.and("isImplicit", HostSearch.entity().getIsImplicit(), SearchCriteria.Op.EQ);
         HostSearch.and("isTagARule", HostSearch.entity().getIsTagARule(), SearchCriteria.Op.EQ);
         HostSearch.done();
 
@@ -46,6 +59,19 @@ public HostTagsDaoImpl() {
         DistinctImplictTagsSearch.and("hostIds", DistinctImplictTagsSearch.entity().getHostId(), SearchCriteria.Op.IN);
         DistinctImplictTagsSearch.and("implicitTags", DistinctImplictTagsSearch.entity().getTag(), SearchCriteria.Op.IN);
         DistinctImplictTagsSearch.done();
+
+        stSearch = createSearchBuilder();
+        stSearch.and("idIN", stSearch.entity().getId(), SearchCriteria.Op.IN);
+        stSearch.done();
+
+        tagIdsearch = createSearchBuilder();
+        tagIdsearch.and("id", tagIdsearch.entity().getId(), SearchCriteria.Op.EQ);
+        tagIdsearch.done();
+
+        ImplicitTagsSearch = createSearchBuilder();
+        ImplicitTagsSearch.and("hostId", ImplicitTagsSearch.entity().getHostId(), SearchCriteria.Op.EQ);
+        ImplicitTagsSearch.and("isImplicit", ImplicitTagsSearch.entity().getIsImplicit(), SearchCriteria.Op.EQ);
+        ImplicitTagsSearch.done();
     }
 
     @Override
@@ -74,6 +100,36 @@ public void deleteTags(long hostId) {
         txn.commit();
     }
 
+    @Override
+    public boolean updateImplicitTags(long hostId, List<String> hostTags) {
+        TransactionLegacy txn = TransactionLegacy.currentTxn();
+        txn.start();
+        SearchCriteria<HostTagVO> sc = ImplicitTagsSearch.create();
+        sc.setParameters("hostId", hostId);
+        sc.setParameters("isImplicit", true);
+        boolean expunged = expunge(sc) > 0;
+        boolean persisted = false;
+        for (String tag : hostTags) {
+            if (StringUtils.isNotBlank(tag)) {
+                HostTagVO vo = new HostTagVO(hostId, tag.trim());
+                vo.setIsImplicit(true);
+                persist(vo);
+                persisted = true;
+            }
+        }
+        txn.commit();
+        return expunged || persisted;
+    }
+
+    @Override
+    public List<HostTagVO> getExplicitHostTags(long hostId) {
+        SearchCriteria<HostTagVO> sc = ImplicitTagsSearch.create();
+        sc.setParameters("hostId", hostId);
+        sc.setParameters("isImplicit", false);
+
+        return search(sc, null);
+    }
+
     @Override
     public List<HostTagVO> findHostRuleTags() {
         SearchCriteria<HostTagVO> sc = HostSearch.create();
@@ -89,6 +145,7 @@ public void persist(long hostId, List<String> hostTags, Boolean isTagARule) {
         txn.start();
         SearchCriteria<HostTagVO> sc = HostSearch.create();
         sc.setParameters("hostId", hostId);
+        sc.setParameters("isImplicit", false);
         expunge(sc);
 
         for (String tag : hostTags) {
@@ -110,4 +167,72 @@ public ConfigKey<?>[] getConfigKeys() {
     public String getConfigComponentName() {
         return HostTagsDaoImpl.class.getSimpleName();
     }
+
+    @Override
+    public HostTagResponse newHostTagResponse(HostTagVO tag) {
+        HostTagResponse tagResponse = new HostTagResponse();
+
+        tagResponse.setName(tag.getTag());
+        tagResponse.setHostId(tag.getHostId());
+        tagResponse.setImplicit(tag.getIsImplicit());
+
+        tagResponse.setObjectName("hosttag");
+
+        return tagResponse;
+    }
+
+    @Override
+    public List<HostTagVO> searchByIds(Long... tagIds) {
+        String batchCfg = _configDao.getValue("detail.batch.query.size");
+
+        final int detailsBatchSize = batchCfg != null ? Integer.parseInt(batchCfg) : 2000;
+
+        // query details by batches
+        List<HostTagVO> tagList = new ArrayList<>();
+        int curr_index = 0;
+
+        if (tagIds.length > detailsBatchSize) {
+            while ((curr_index + detailsBatchSize) <= tagIds.length) {
+                Long[] ids = new Long[detailsBatchSize];
+
+                for (int k = 0, j = curr_index; j < curr_index + detailsBatchSize; j++, k++) {
+                    ids[k] = tagIds[j];
+                }
+
+                SearchCriteria<HostTagVO> sc = stSearch.create();
+
+                sc.setParameters("idIN", (Object[])ids);
+
+                List<HostTagVO> vms = searchIncludingRemoved(sc, null, null, false);
+
+                if (vms != null) {
+                    tagList.addAll(vms);
+                }
+
+                curr_index += detailsBatchSize;
+            }
+        }
+
+        if (curr_index < tagIds.length) {
+            int batch_size = (tagIds.length - curr_index);
+            // set the ids value
+            Long[] ids = new Long[batch_size];
+
+            for (int k = 0, j = curr_index; j < curr_index + batch_size; j++, k++) {
+                ids[k] = tagIds[j];
+            }
+
+            SearchCriteria<HostTagVO> sc = stSearch.create();
+
+            sc.setParameters("idIN", (Object[])ids);
+
+            List<HostTagVO> tags = searchIncludingRemoved(sc, null, null, false);
+
+            if (tags != null) {
+                tagList.addAll(tags);
+            }
+        }
+
+        return tagList;
+    }
 }

engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml

@@ -187,7 +187,6 @@
   <bean id="storageNetworkIpAddressDaoImpl" class="com.cloud.dc.dao.StorageNetworkIpAddressDaoImpl" />
   <bean id="storageNetworkIpRangeDaoImpl" class="com.cloud.dc.dao.StorageNetworkIpRangeDaoImpl" />
   <bean id="storagePoolJoinDaoImpl" class="com.cloud.api.query.dao.StoragePoolJoinDaoImpl" />
-  <bean id="hostTagDaoImpl" class="com.cloud.api.query.dao.HostTagDaoImpl" />
   <bean id="storagePoolWorkDaoImpl" class="com.cloud.storage.dao.StoragePoolWorkDaoImpl" />
   <bean id="uploadDaoImpl" class="com.cloud.storage.dao.UploadDaoImpl" />
   <bean id="usageDaoImpl" class="com.cloud.usage.dao.UsageDaoImpl" />

engine/schema/src/main/resources/META-INF/db/schema-41900to42000.sql

@@ -79,3 +79,6 @@ CREATE TABLE IF NOT EXISTS `cloud_usage`.`quota_email_configuration`(
     PRIMARY KEY (`account_id`, `email_template_id`),
     CONSTRAINT `FK_quota_email_configuration_account_id` FOREIGN KEY (`account_id`) REFERENCES `cloud_usage`.`quota_account`(`account_id`),
     CONSTRAINT `FK_quota_email_configuration_email_template_id` FOREIGN KEY (`email_template_id`) REFERENCES `cloud_usage`.`quota_email_templates`(`id`));
+
+-- Add `is_implicit` column to `host_tags` table
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.host_tags', 'is_implicit', 'int(1) UNSIGNED NOT NULL DEFAULT 0 COMMENT "If host tag is implicit or explicit" ');

engine/schema/src/main/resources/META-INF/db/views/cloud.host_view.sql

@@ -53,7 +53,9 @@ SELECT
     host_pod_ref.uuid pod_uuid,
     host_pod_ref.name pod_name,
     GROUP_CONCAT(DISTINCT(host_tags.tag)) AS tag,
-    `host_tags`.`is_tag_a_rule` AS `is_tag_a_rule`,
+    GROUP_CONCAT(DISTINCT(explicit_host_tags.tag)) AS explicit_tag,
+    GROUP_CONCAT(DISTINCT(implicit_host_tags.tag)) AS implicit_tag,
+    `explicit_host_tags`.`is_tag_a_rule` AS `is_tag_a_rule`,
     guest_os_category.id guest_os_category_id,
     guest_os_category.uuid guest_os_category_uuid,
     guest_os_category.name guest_os_category_name,
@@ -89,6 +91,10 @@ FROM
         LEFT JOIN
     `cloud`.`host_tags` ON host_tags.host_id = host.id
         LEFT JOIN
+    `cloud`.`host_tags` AS explicit_host_tags ON explicit_host_tags.host_id = host.id AND explicit_host_tags.is_implicit = 0
+        LEFT JOIN
+    `cloud`.`host_tags` AS implicit_host_tags ON implicit_host_tags.host_id = host.id AND implicit_host_tags.is_implicit = 1
+        LEFT JOIN
     `cloud`.`op_host_capacity` mem_caps ON host.id = mem_caps.host_id
         AND mem_caps.capacity_type = 0
         LEFT JOIN