FR84: Add "allocation" and "disable" thresholds per storage pool (apache#526)

Introduces multiple scope configuration parameters by changing the column `scope` of table `configuration` from string to bitmask.

Following configuration parameters can now be configured at a Storage Pool scope in addition to the Zone scope.

1. **pool.storage.capacity.disablethreshold**: Percentage (as a value between 0 and 1) of storage utilization or used space relative to the total storage capacity above which the storage pool is disabled for new allocations. migration and resize operations.

2. **pool.storage.allocated.capacity.disablethreshold**: Percentage (as a value between 0 and 1) of allocated storage space relative to total overprovisioned storage capacity above which the storage pool is disabled for new allocations, migration and resize operations.

3. **pool.storage.allocated.resize.capacity.disablethreshold**: Percentage (as a value between 0 and 1) of allocated storage space relative to total overprovisioned storage capacity above which the storage pool is disabled for volume resize. This is applicable only when volume.resize.allowed.beyond.allocation is set to true

4. **volume.resize.allowed.beyond.allocation**: Specifies whether a volume can be resized beyond the pool capacity allocation disable threshold (pool.storage.allocated.capacity.disablethreshold) but not exceeding the resize capacity disable threshold (pool.storage.allocated.resize.capacity.disablethreshold). Please note that resize due to change of disk offering still honours pool capacity allocation disable threshold (pool.storage.allocated.capacity.disablethreshold).
If set, the granular scope (storage pool) overrides the value set for the broader scope (zone). 

If set, the granular scope (storage pool) overrides the value set for the broader scope (zone). 

The operator will see these configurations in the Settings tab under zone as well as under primary storage. listConfigurations storageid=<storage_id> | zoneid=<zone_id> and updateConfigurations storage_id=<storage_id> | zoneid=<zone_id> will also work.

Upstream PR: apache#10300

---------

Co-authored-by: Harikrishna Patnala <[email protected]>

Author: abh1sar

engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java

@@ -16,6 +16,8 @@
 // under the License.
 package com.cloud.capacity;
 
+import java.util.List;
+
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 
@@ -67,7 +69,7 @@ public interface CapacityManager {
                     "0.85",
                     "Percentage (as a value between 0 and 1) of storage utilization above which allocators will disable using the pool for low storage available.",
                     true,
-                    ConfigKey.Scope.Zone);
+                    List.of(ConfigKey.Scope.StoragePool, ConfigKey.Scope.Zone));
     static final ConfigKey<Double> StorageOverprovisioningFactor =
             new ConfigKey<>(
                     "Storage",
@@ -85,7 +87,7 @@ public interface CapacityManager {
                     "0.85",
                     "Percentage (as a value between 0 and 1) of allocated storage utilization above which allocators will disable using the pool for low allocated storage available.",
                     true,
-                    ConfigKey.Scope.Zone);
+                    List.of(ConfigKey.Scope.StoragePool, ConfigKey.Scope.Zone));
     static final ConfigKey<Boolean> StorageOperationsExcludeCluster =
             new ConfigKey<>(
                     Boolean.class,
@@ -125,7 +127,7 @@ public interface CapacityManager {
                     "Percentage (as a value between 0 and 1) of allocated storage utilization above which allocators will disable using the pool for volume resize. " +
                             "This is applicable only when volume.resize.allowed.beyond.allocation is set to true.",
                     true,
-                    ConfigKey.Scope.Zone);
+                    List.of(ConfigKey.Scope.StoragePool, ConfigKey.Scope.Zone));
 
     ConfigKey<Integer> CapacityCalculateWorkers = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Integer.class,
             "capacity.calculate.workers", "1",

engine/components-api/src/main/java/com/cloud/storage/StorageManager.java

@@ -211,7 +211,7 @@ public interface StorageManager extends StorageService {
     ConfigKey<Boolean> AllowVolumeReSizeBeyondAllocation = new ConfigKey<Boolean>("Advanced", Boolean.class, "volume.resize.allowed.beyond.allocation", "false",
             "Determines whether volume size can exceed the pool capacity allocation disable threshold (pool.storage.allocated.capacity.disablethreshold) " +
                     "when resize a volume upto resize capacity disable threshold (pool.storage.allocated.resize.capacity.disablethreshold)",
-            true, ConfigKey.Scope.Zone);
+            true, List.of(ConfigKey.Scope.StoragePool, ConfigKey.Scope.Zone));
 
     ConfigKey<Integer> StoragePoolHostConnectWorkers = new ConfigKey<>("Storage", Integer.class,
             "storage.pool.host.connect.workers", "1",

engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDaoImpl.java

@@ -22,18 +22,27 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import javax.inject.Inject;
+
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.ConfigKey.Scope;
 import org.apache.cloudstack.framework.config.ScopedConfigStorage;
 import org.apache.commons.collections.CollectionUtils;
 
+import com.cloud.dc.dao.ClusterDao;
+import com.cloud.org.Cluster;
+import com.cloud.utils.Pair;
 import com.cloud.utils.crypt.DBEncryptionUtil;
 import com.cloud.utils.db.GenericDaoBase;
 import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
 import com.cloud.utils.db.TransactionLegacy;
 
 public class ClusterDetailsDaoImpl extends GenericDaoBase<ClusterDetailsVO, Long> implements ClusterDetailsDao, ScopedConfigStorage {
+
+    @Inject
+    ClusterDao clusterDao;
+
     protected final SearchBuilder<ClusterDetailsVO> ClusterSearch;
     protected final SearchBuilder<ClusterDetailsVO> DetailSearch;
 
@@ -180,4 +189,13 @@ private String getCpuMemoryOvercommitRatio(String name) {
 
         return name;
     }
+
+    @Override
+    public Pair<Scope, Long> getParentScope(long id) {
+        Cluster cluster = clusterDao.findById(id);
+        if (cluster == null) {
+            return null;
+        }
+        return new Pair<>(getScope().getParent(), cluster.getDataCenterId());
+    }
 }

engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java

@@ -30,6 +30,8 @@
 import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 
+import com.cloud.utils.Pair;
+
 public class StoragePoolDetailsDaoImpl extends ResourceDetailsDaoBase<StoragePoolDetailVO> implements StoragePoolDetailsDao, ScopedConfigStorage {
 
     @Inject
@@ -57,4 +59,17 @@ public void addDetail(long resourceId, String key, String value, boolean display
         }
         super.addDetail(new StoragePoolDetailVO(resourceId, key, value, display));
     }
+
+    @Override
+    public Pair<Scope, Long> getParentScope(long id) {
+        StoragePoolVO pool = _storagePoolDao.findById(id);
+        if (pool != null) {
+            if (pool.getClusterId() != null) {
+                return new Pair<>(getScope().getParent(), pool.getClusterId());
+            } else {
+                return new Pair<>(ConfigKey.Scope.Zone, pool.getDataCenterId());
+            }
+        }
+        return null;
+    }
 }

engine/schema/src/main/java/com/cloud/upgrade/ConfigurationGroupsAggregator.java

@@ -53,7 +53,7 @@ public ConfigurationGroupsAggregator() {
 
     public void updateConfigurationGroups() {
         LOG.debug("Updating configuration groups");
-        List<ConfigurationVO> configs =  configDao.listAllIncludingRemoved();
+        List<ConfigurationVO> configs = configDao.searchPartialConfigurations();
         if (CollectionUtils.isEmpty(configs)) {
             return;
         }

engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java

@@ -90,6 +90,36 @@ public String generateIndexName(String tableName, String... columnNames) {
         return String.format("i_%s__%s", tableName, StringUtils.join(columnNames, "__"));
     }
 
+    public String getColumnType(Connection conn, String tableName, String columnName) {
+        try (PreparedStatement pstmt = conn.prepareStatement(String.format("DESCRIBE %s %s", tableName, columnName));){
+            ResultSet rs = pstmt.executeQuery();
+            if (rs.next()) {
+                return rs.getString("Type");
+            }
+        } catch (SQLException e) {
+            s_logger.warn("Type for column " + columnName + " can not be retrieved in " + tableName + " ignoring exception: " + e.getMessage());
+        }
+        return null;
+    }
+
+    public void addColumn(Connection conn, String tableName, String columnName, String columnDefinition) {
+        try (PreparedStatement pstmt = conn.prepareStatement(String.format("ALTER TABLE %s ADD COLUMN %s %s", tableName, columnName, columnDefinition));){
+            pstmt.executeUpdate();
+            s_logger.debug("Column " + columnName + " is added successfully from the table " + tableName);
+        } catch (SQLException e) {
+            s_logger.warn("Unable to add column " + columnName + " to table " + tableName + " due to exception ", e);
+        }
+    }
+
+    public void changeColumn(Connection conn, String tableName, String oldColumnName, String newColumnName, String columnDefinition) {
+        try (PreparedStatement pstmt = conn.prepareStatement(String.format("ALTER TABLE %s CHANGE COLUMN %s %s %s", tableName, oldColumnName, newColumnName, columnDefinition));){
+            pstmt.executeUpdate();
+            s_logger.debug("Column " + oldColumnName + " is changed successfully to " + newColumnName + " from the table " + tableName);
+        } catch (SQLException e) {
+            s_logger.warn("Unable to add column " + oldColumnName + " to " + newColumnName + " from the table " + tableName + " due to exception", e);
+        }
+    }
+
     public boolean indexExists(Connection conn, String tableName, String indexName) {
         try (PreparedStatement pstmt = conn.prepareStatement(String.format("SHOW INDEXES FROM %s where Key_name = \"%s\"", tableName, indexName))) {
             ResultSet result = pstmt.executeQuery();

engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java

@@ -58,4 +58,20 @@ public static void dropTableColumnsIfExist(Connection conn, String tableName, Li
         }
     }
 
+    public static String getTableColumnType(Connection conn, String tableName, String columnName) {
+        return dao.getColumnType(conn, tableName, columnName);
+    }
+
+    public static void addTableColumnIfNotExist(Connection conn, String tableName, String columnName, String columnDefinition) {
+        if (!dao.columnExists(conn, tableName, columnName)) {
+            dao.addColumn(conn, tableName, columnName, columnDefinition);
+        }
+    }
+
+    public static void changeTableColumnIfNotExist(Connection conn, String tableName, String oldColumnName, String newColumnName, String columnDefinition) {
+        if (dao.columnExists(conn, tableName, oldColumnName)) {
+            dao.changeColumn(conn, tableName, oldColumnName, newColumnName, columnDefinition);
+        }
+    }
+
 }

engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41811to41812.java

@@ -16,11 +16,17 @@
 // under the License.
 package com.cloud.upgrade.dao;
 
+import com.cloud.utils.db.TransactionLegacy;
 import com.cloud.utils.exception.CloudRuntimeException;
+
+import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.log4j.Logger;
 
 import java.io.InputStream;
 import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.List;
 
 public class Upgrade41811to41812 implements DbUpgrade {
     final static Logger LOG = Logger.getLogger(Upgrade41811to41812.class);
@@ -53,6 +59,7 @@ public InputStream[] getPrepareScripts() {
 
     @Override
     public void performDataMigration(Connection conn) {
+        migrateConfigurationScopeToBitmask(conn);
     }
 
 
@@ -66,4 +73,35 @@ public InputStream[] getCleanupScripts() {
 
         return new InputStream[]{script};
     }
+
+    protected void migrateConfigurationScopeToBitmask(Connection conn) {
+        String scopeDataType = DbUpgradeUtils.getTableColumnType(conn, "configuration", "scope");
+        LOG.info("Data type of the column scope of table configuration is " + scopeDataType);
+        if (!"varchar(255)".equals(scopeDataType)) {
+            return;
+        }
+        DbUpgradeUtils.addTableColumnIfNotExist(conn, "configuration", "new_scope", "BIGINT DEFAULT 0");
+        migrateExistingConfigurationScopeValues(conn);
+        DbUpgradeUtils.dropTableColumnsIfExist(conn, "configuration", List.of("scope"));
+        DbUpgradeUtils.changeTableColumnIfNotExist(conn, "configuration", "new_scope", "scope", "BIGINT NOT NULL DEFAULT 0 COMMENT 'Bitmask for scope(s) of this parameter'");
+    }
+
+    protected void migrateExistingConfigurationScopeValues(Connection conn) {
+        StringBuilder sql = new StringBuilder("UPDATE configuration\n" +
+                "SET new_scope = " +
+                "    CASE ");
+        for (ConfigKey.Scope scope : ConfigKey.Scope.values()) {
+            sql.append("        WHEN scope = '").append(scope.name()).append("' THEN ").append(scope.getBitValue()).append(" ");
+        }
+        sql.append("        ELSE 0 " +
+                "    END " +
+                "WHERE scope IS NOT NULL;");
+        TransactionLegacy txn = TransactionLegacy.currentTxn();
+        try (PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql.toString())) {
+            pstmt.executeUpdate();
+        } catch (SQLException e) {
+            LOG.error("Failed to migrate existing configuration scope values to bitmask", e);
+            throw new CloudRuntimeException(String.format("Failed to migrate existing configuration scope values to bitmask due to: %s", e.getMessage()));
+        }
+    }
 }

engine/schema/src/main/java/com/cloud/user/AccountDetailsDaoImpl.java

@@ -33,6 +33,7 @@
 import com.cloud.domain.dao.DomainDao;
 import com.cloud.domain.dao.DomainDetailsDao;
 import com.cloud.user.dao.AccountDao;
+import com.cloud.utils.Pair;
 import com.cloud.utils.db.GenericDaoBase;
 import com.cloud.utils.db.QueryBuilder;
 import com.cloud.utils.db.SearchBuilder;
@@ -151,4 +152,13 @@ public String getConfigValue(long id, String key) {
         }
         return value;
     }
+
+    @Override
+    public Pair<Scope, Long> getParentScope(long id) {
+        Account account = _accountDao.findById(id);
+        if (account == null) {
+            return null;
+        }
+        return new Pair<>(getScope().getParent(), account.getDomainId());
+    }
 }

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDetailsDaoImpl.java

@@ -20,13 +20,17 @@
 import java.util.List;
 import java.util.Map;
 
+import javax.inject.Inject;
+
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.ConfigKey.Scope;
 import org.apache.cloudstack.framework.config.ScopedConfigStorage;
 import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase;
 import org.springframework.stereotype.Component;
 
+import com.cloud.storage.ImageStore;
+import com.cloud.utils.Pair;
 import com.cloud.utils.crypt.DBEncryptionUtil;
 import com.cloud.utils.db.QueryBuilder;
 import com.cloud.utils.db.SearchBuilder;
@@ -36,6 +40,8 @@
 
 @Component
 public class ImageStoreDetailsDaoImpl extends ResourceDetailsDaoBase<ImageStoreDetailVO> implements ImageStoreDetailsDao, ScopedConfigStorage {
+    @Inject
+    ImageStoreDao imageStoreDao;
 
     protected final SearchBuilder<ImageStoreDetailVO> storeSearch;
 
@@ -115,4 +121,13 @@ public void addDetail(long resourceId, String key, String value, boolean display
         super.addDetail(new ImageStoreDetailVO(resourceId, key, value, display));
     }
 
+    @Override
+    public Pair<Scope, Long> getParentScope(long id) {
+        ImageStore store = imageStoreDao.findById(id);
+        if (store == null) {
+            return null;
+        }
+        return new Pair<>(getScope().getParent(), store.getDataCenterId());
+    }
+
 }

engine/schema/src/test/java/com/cloud/upgrade/ConfigurationGroupsAggregatorTest.java

@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.upgrade;
+
+import static org.mockito.Mockito.when;
+
+import java.util.Collections;
+
+import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.cloudstack.framework.config.dao.ConfigurationGroupDao;
+import org.apache.cloudstack.framework.config.dao.ConfigurationSubGroupDao;
+import org.apache.cloudstack.framework.config.impl.ConfigurationSubGroupVO;
+import org.apache.cloudstack.framework.config.impl.ConfigurationVO;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ConfigurationGroupsAggregatorTest {
+    @InjectMocks
+    private ConfigurationGroupsAggregator configurationGroupsAggregator = new ConfigurationGroupsAggregator();
+
+    @Mock
+    private ConfigurationDao configDao;
+
+    @Mock
+    private ConfigurationGroupDao configGroupDao;
+
+    @Mock
+    private ConfigurationSubGroupDao configSubGroupDao;
+
+    @Test
+    public void testUpdateConfigurationGroups() {
+        ConfigurationVO config = new ConfigurationVO("Advanced", "DEFAULT", "management-server",
+                "test.config.name", null, "description");
+        config.setGroupId(1L);
+        config.setSubGroupId(1L);
+
+        when(configDao.searchPartialConfigurations()).thenReturn(Collections.singletonList(config));
+
+        ConfigurationSubGroupVO configSubGroup = Mockito.mock(ConfigurationSubGroupVO.class);
+        when(configSubGroupDao.findByName("name")).thenReturn(configSubGroup);
+        Mockito.when(configSubGroup.getId()).thenReturn(10L);
+        Mockito.when(configSubGroup.getGroupId()).thenReturn(5L);
+
+        configurationGroupsAggregator.updateConfigurationGroups();
+
+        Assert.assertEquals(Long.valueOf(5), config.getGroupId());
+        Assert.assertEquals(Long.valueOf(10), config.getSubGroupId());
+        Mockito.verify(configDao, Mockito.times(1)).persist(config);
+    }
+}