Add Resource Limits to Backups and Object Storage (#10017)

Doc PR : apache/cloudstack-documentation#461
This PR fixes #8638

== Description

Four new Resource Types have been added. Admin can configure corresponding resource limits for the tenants at different levels (domain, account, project) 
User dashboard's Storage section will show the new resources, their limits and current usage.

1. backup - No. of backups used by the account
2. backup_storage - Backup storage allocated for the account
3. bucket - No. of buckets used by the accounts
4. object_storage - Object storage allocated for the account.

Some other related changes done to BnR framework:

1. Maximum number of Backups to retain can be specified while creating Backup schedules, similar to Scheduled snapshots.
2. Oldest Scheduled backup of the same interval type will be deleted once the number reaches the configured max Backups value.
3. Code refactor: Moved syncBackups method from BackupProvider to the framework BackupManagerImpl, as it is a common functionality and all providers were using duplicated code.

Changes done to the Object Storage Framework

1. Quota parameter is made mandatory while creating a bucket. Bucket quota is considered to be the allocated space and will be used to enforce Resource limits.

== Schema Changes:

1. New Column `max_backups` added to `backup_schedule` table
4. New Column `backup_interval_type` added to `backups` table

== Api Changes:

1. createBackup: new Parameter `scheduleid`. It should be specified whenever a scheduled backup is created. This will translate to the `backup_interval_type` in the `backups` table.
3. createBackupScheduke: new Parameter `max_backups`. To specify maximum number of backups to retain for the given schedule.

== Configurations:

|Setting |Scope |Default Value |Description|
|-------|--------|--------------|-----------|
|backup.max.hourly |Global |8 |Maximum recurring hourly backups to be retained for an instance|
|backup.max.daily |Global |8 |Maximum recurring daily backups to be retained for an instance|
|backup.max.weekly |Global |8 |Maximum recurring weekly backups to be retained for an instance|
|backup.max.monthly |Global |8 |Maximum recurring monthly backups to be retained for an instance|
|max.account.backups| Global| 20 | The default maximum number of backups that can be created for an account|
|max.account.backup.storage| Global| 400 | The default maximum backup storage space (in GiB) that can be used for an account|
|max.domain.backups| Global| 40 | The default maximum number of backups that can be created for an domain|
|max.domain.backup.storage| Global| 800 | The default maximum backup storage space (in GiB) that can be used for an domain|
|max.project.backups| Global| 20 | The default maximum number of backups that can be created for an project|
|max.project.backup.storage| Global| 400 | The default maximum backup storage space (in GiB) that can be used for an project|

|Setting |Scope |Default Value |Description|
|-------|--------|--------------|-----------|
|max.account.buckets| Global| 20 | The default maximum number of buckets that can be created for an account|
|max.account.object.storage| Global| 400 | The default maximum object storage space (in GiB) that can be used for an account|
|max.domain.buckets| Global| 40 | The default maximum number of buckets that can be created for an domain|
|max.domain.object.storage| Global| 800 | The default maximum object storage space (in GiB) that can be used for an domain|
|max.project.buckets| Global| 20 | The default maximum number of buckets that can be created for an project|
|max.project.object.storage| Global| 400 | The default maximum object storage space (in GiB) that can be used for an project|


Co-authored-by: Daan Hoogland <[email protected]>
Co-authored-by: Lucas Martins <[email protected]>
Co-authored-by: Lucas Martins <[email protected]>
Co-authored-by: Pearl Dsilva <[email protected]>
Co-authored-by: Rohit Yadav <[email protected]>

Author: 6 people

api/src/main/java/com/cloud/configuration/Resource.java

@@ -21,7 +21,7 @@ public interface Resource {
     short RESOURCE_UNLIMITED = -1;
     String UNLIMITED = "Unlimited";
 
-    enum ResourceType { // Primary and Secondary storage are allocated_storage and not the physical storage.
+    enum ResourceType { // All storage type resources are allocated_storage and not the physical storage.
         user_vm("user_vm", 0),
         public_ip("public_ip", 1),
         volume("volume", 2),
@@ -33,7 +33,11 @@ enum ResourceType { // Primary and Secondary storage are allocated_storage and n
         cpu("cpu", 8),
         memory("memory", 9),
         primary_storage("primary_storage", 10),
-        secondary_storage("secondary_storage", 11);
+        secondary_storage("secondary_storage", 11),
+        backup("backup", 12),
+        backup_storage("backup_storage", 13),
+        bucket("bucket", 14),
+        object_storage("object_storage", 15);
 
         private String name;
         private int ordinal;
@@ -62,6 +66,10 @@ public static ResourceType fromOrdinal(int ordinal) {
             }
             return null;
         }
+
+        public static Boolean isStorageType(ResourceType type) {
+            return (type == primary_storage || type == secondary_storage || type == backup_storage || type == object_storage);
+        }
     }
 
     public static class ResourceOwnerType {

api/src/main/java/com/cloud/event/EventTypes.java

@@ -785,6 +785,9 @@ public class EventTypes {
     public static final String EVENT_SHAREDFS_EXPUNGE = "SHAREDFS.EXPUNGE";
     public static final String EVENT_SHAREDFS_RECOVER = "SHAREDFS.RECOVER";
 
+    // Resource Limit
+    public static final String EVENT_RESOURCE_LIMIT_UPDATE = "RESOURCE.LIMIT.UPDATE";
+
     static {
 
         // TODO: need a way to force author adding event types to declare the entity details as well, with out braking

api/src/main/java/com/cloud/storage/VolumeApiService.java

@@ -190,4 +190,6 @@ Volume updateVolume(long volumeId, String path, String state, Long storageId,
     boolean stateTransitTo(Volume vol, Volume.Event event) throws NoTransitionException;
 
     Pair<String, String> checkAndRepairVolume(CheckAndRepairVolumeCmd cmd) throws ResourceAllocationException;
+
+    Long getVolumePhysicalSize(Storage.ImageFormat format, String path, String chainInfo);
 }

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -51,9 +51,15 @@ public class ApiConstants {
     public static final String AVAILABLE = "available";
     public static final String AVAILABLE_SUBNETS = "availablesubnets";
     public static final String AVAILABLE_VIRTUAL_MACHINE_COUNT = "availablevirtualmachinecount";
+    public static final String BACKUP_AVAILABLE = "backupavailable";
     public static final String BACKUP_ID = "backupid";
+    public static final String BACKUP_LIMIT = "backuplimit";
     public static final String BACKUP_OFFERING_NAME = "backupofferingname";
     public static final String BACKUP_OFFERING_ID = "backupofferingid";
+    public static final String BACKUP_STORAGE_AVAILABLE = "backupstorageavailable";
+    public static final String BACKUP_STORAGE_LIMIT = "backupstoragelimit";
+    public static final String BACKUP_STORAGE_TOTAL = "backupstoragetotal";
+    public static final String BACKUP_TOTAL = "backuptotal";
     public static final String BASE64_IMAGE = "base64image";
     public static final String BGP_PEERS = "bgppeers";
     public static final String BGP_PEER_IDS = "bgppeerids";
@@ -322,6 +328,7 @@ public class ApiConstants {
     public static final String MAC_ADDRESS = "macaddress";
     public static final String MAX = "max";
     public static final String MAX_SNAPS = "maxsnaps";
+    public static final String MAX_BACKUPS = "maxbackups";
     public static final String MAX_CPU_NUMBER = "maxcpunumber";
     public static final String MAX_MEMORY = "maxmemory";
     public static final String MIN_CPU_NUMBER = "mincpunumber";
@@ -436,6 +443,7 @@ public class ApiConstants {
     public static final String QUALIFIERS = "qualifiers";
     public static final String QUERY_FILTER = "queryfilter";
     public static final String SCHEDULE = "schedule";
+    public static final String SCHEDULE_ID = "scheduleid";
     public static final String SCOPE = "scope";
     public static final String SEARCH_BASE = "searchbase";
     public static final String SECONDARY_IP = "secondaryip";
@@ -1148,7 +1156,6 @@ public class ApiConstants {
     public static final String MTU = "mtu";
     public static final String AUTO_ENABLE_KVM_HOST = "autoenablekvmhost";
     public static final String LIST_APIS = "listApis";
-    public static final String OBJECT_STORAGE_ID = "objectstorageid";
     public static final String VERSIONING = "versioning";
     public static final String OBJECT_LOCKING = "objectlocking";
     public static final String ENCRYPTION = "encryption";
@@ -1162,7 +1169,6 @@ public class ApiConstants {
     public static final String DISK_PATH = "diskpath";
     public static final String IMPORT_SOURCE = "importsource";
     public static final String TEMP_PATH = "temppath";
-    public static final String OBJECT_STORAGE = "objectstore";
     public static final String HEURISTIC_RULE = "heuristicrule";
     public static final String HEURISTIC_TYPE_VALID_OPTIONS = "Valid options are: ISO, SNAPSHOT, TEMPLATE and VOLUME.";
     public static final String MANAGEMENT = "management";
@@ -1190,11 +1196,20 @@ public class ApiConstants {
     public static final String SHAREDFSVM_MIN_CPU_COUNT = "sharedfsvmmincpucount";
     public static final String SHAREDFSVM_MIN_RAM_SIZE = "sharedfsvmminramsize";
 
+    // Object Storage related
+    public static final String BUCKET_AVAILABLE = "bucketavailable";
+    public static final String BUCKET_LIMIT = "bucketlimit";
+    public static final String BUCKET_TOTAL = "buckettotal";
+    public static final String OBJECT_STORAGE_ID = "objectstorageid";
+    public static final String OBJECT_STORAGE = "objectstore";
+    public static final String OBJECT_STORAGE_AVAILABLE = "objectstorageavailable";
+    public static final String OBJECT_STORAGE_LIMIT = "objectstoragelimit";
+    public static final String OBJECT_STORAGE_TOTAL = "objectstoragetotal";
+
     public static final String PARAMETER_DESCRIPTION_ACTIVATION_RULE = "Quota tariff's activation rule. It can receive a JS script that results in either " +
             "a boolean or a numeric value: if it results in a boolean value, the tariff value will be applied according to the result; if it results in a numeric value, the " +
             "numeric value will be applied; if the result is neither a boolean nor a numeric value, the tariff will not be applied. If the rule is not informed, the tariff " +
             "value will be applied.";
-
     public static final String PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS = "The recommended format is \"yyyy-MM-dd'T'HH:mm:ssZ\" (e.g.: \"2023-01-01T12:00:00+0100\"); " +
             "however, the following formats are also accepted: \"yyyy-MM-dd HH:mm:ss\" (e.g.: \"2023-01-01 12:00:00\") and \"yyyy-MM-dd\" (e.g.: \"2023-01-01\" - if the time is not " +
             "added, it will be interpreted as \"00:00:00\"). If the recommended format is not used, the date will be considered in the server timezone.";

api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java

@@ -19,6 +19,7 @@
 
 import javax.inject.Inject;
 
+import com.cloud.storage.Snapshot;
 import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiCommandResourceType;
@@ -27,6 +28,7 @@
 import org.apache.cloudstack.api.BaseAsyncCreateCmd;
 import org.apache.cloudstack.api.Parameter;
 import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.BackupScheduleResponse;
 import org.apache.cloudstack.api.response.SuccessResponse;
 import org.apache.cloudstack.api.response.UserVmResponse;
 import org.apache.cloudstack.backup.BackupManager;
@@ -60,6 +62,13 @@ public class CreateBackupCmd extends BaseAsyncCreateCmd {
             description = "ID of the VM")
     private Long vmId;
 
+    @Parameter(name = ApiConstants.SCHEDULE_ID,
+            type = CommandType.LONG,
+            entityType = BackupScheduleResponse.class,
+            description = "backup schedule ID of the VM, if this is null, it indicates that it is a manual backup.",
+            since = "4.21.0")
+    private Long scheduleId;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -68,14 +77,22 @@ public Long getVmId() {
         return vmId;
     }
 
+    public Long getScheduleId() {
+        if (scheduleId != null) {
+            return scheduleId;
+        } else {
+            return Snapshot.MANUAL_POLICY_ID;
+        }
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////
 
     @Override
     public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
         try {
-            boolean result = backupManager.createBackup(getVmId());
+            boolean result = backupManager.createBackup(getVmId(), getScheduleId());
             if (result) {
                 SuccessResponse response = new SuccessResponse(getCommandName());
                 response.setResponseName(getCommandName());

api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java

@@ -75,6 +75,12 @@ public class CreateBackupScheduleCmd extends BaseCmd {
             description = "Specifies a timezone for this command. For more information on the timezone parameter, see TimeZone Format.")
     private String timezone;
 
+    @Parameter(name = ApiConstants.MAX_BACKUPS,
+            type = CommandType.INTEGER,
+            description = "maximum number of backups to retain",
+            since = "4.21.0")
+    private Integer maxBackups;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -95,6 +101,10 @@ public String getTimezone() {
         return timezone;
     }
 
+    public Integer getMaxBackups() {
+        return maxBackups;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java

@@ -72,7 +72,7 @@ public class CreateBucketCmd extends BaseAsyncCreateCmd implements UserCmd {
             description = "Id of the Object Storage Pool where bucket is created")
     private long objectStoragePoolId;
 
-    @Parameter(name = ApiConstants.QUOTA, type = CommandType.INTEGER,description = "Bucket Quota in GB")
+    @Parameter(name = ApiConstants.QUOTA, type = CommandType.INTEGER, required = true, description = "Bucket Quota in GiB")
     private Integer quota;
 
     @Parameter(name = ApiConstants.ENCRYPTION, type = CommandType.BOOLEAN, description = "Enable bucket encryption")

api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java

@@ -56,7 +56,7 @@ public class UpdateBucketCmd extends BaseCmd {
     @Parameter(name = ApiConstants.POLICY, type = CommandType.STRING, description = "Bucket Access Policy")
     private String policy;
 
-    @Parameter(name = ApiConstants.QUOTA, type = CommandType.INTEGER,description = "Bucket Quota in GB")
+    @Parameter(name = ApiConstants.QUOTA, type = CommandType.INTEGER, description = "Bucket Quota in GiB")
     private Integer quota;
 
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java

@@ -127,6 +127,30 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou
     @Param(description = "the total number of snapshots available for this account")
     private String snapshotAvailable;
 
+    @SerializedName(ApiConstants.BACKUP_LIMIT)
+    @Param(description = "the total number of backups which can be stored by this account", since = "4.21.0")
+    private String backupLimit;
+
+    @SerializedName(ApiConstants.BACKUP_TOTAL)
+    @Param(description = "the total number of backups stored by this account", since = "4.21.0")
+    private Long backupTotal;
+
+    @SerializedName(ApiConstants.BACKUP_AVAILABLE)
+    @Param(description = "the total number of backups available to this account", since = "4.21.0")
+    private String backupAvailable;
+
+    @SerializedName(ApiConstants.BACKUP_STORAGE_LIMIT)
+    @Param(description = "the total backup storage space (in GiB) the account can own", since = "4.21.0")
+    private String backupStorageLimit;
+
+    @SerializedName(ApiConstants.BACKUP_STORAGE_TOTAL)
+    @Param(description = "the total backup storage space (in GiB) owned by the account", since = "4.21.0")
+    private Long backupStorageTotal;
+
+    @SerializedName(ApiConstants.BACKUP_STORAGE_AVAILABLE)
+    @Param(description = "the total backup storage space (in GiB) available to the account", since = "4.21.0")
+    private String backupStorageAvailable;
+
     @SerializedName("templatelimit")
     @Param(description = "the total number of templates which can be created by this account")
     private String templateLimit;
@@ -231,6 +255,30 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou
     @Param(description = "the total secondary storage space (in GiB) available to be used for this account", since = "4.2.0")
     private String secondaryStorageAvailable;
 
+    @SerializedName(ApiConstants.BUCKET_LIMIT)
+    @Param(description = "the total number of buckets which can be stored by this account", since = "4.21.0")
+    private String bucketLimit;
+
+    @SerializedName(ApiConstants.BUCKET_TOTAL)
+    @Param(description = "the total number of buckets stored by this account", since = "4.21.0")
+    private Long bucketTotal;
+
+    @SerializedName(ApiConstants.BUCKET_AVAILABLE)
+    @Param(description = "the total number of buckets available to this account", since = "4.21.0")
+    private String bucketAvailable;
+
+    @SerializedName(ApiConstants.OBJECT_STORAGE_LIMIT)
+    @Param(description = "the total object storage space (in GiB) the account can own", since = "4.21.0")
+    private String objectStorageLimit;
+
+    @SerializedName(ApiConstants.OBJECT_STORAGE_TOTAL)
+    @Param(description = "the total object storage space (in GiB) owned by the account", since = "4.21.0")
+    private Long objectStorageTotal;
+
+    @SerializedName(ApiConstants.OBJECT_STORAGE_AVAILABLE)
+    @Param(description = "the total object storage space (in GiB) available to the account", since = "4.21.0")
+    private String objectStorageAvailable;
+
     @SerializedName(ApiConstants.STATE)
     @Param(description = "the state of the account")
     private String state;
@@ -386,6 +434,36 @@ public void setSnapshotAvailable(String snapshotAvailable) {
         this.snapshotAvailable = snapshotAvailable;
     }
 
+    @Override
+    public void setBackupLimit(String backupLimit) {
+        this.backupLimit = backupLimit;
+    }
+
+    @Override
+    public void setBackupTotal(Long backupTotal) {
+        this.backupTotal = backupTotal;
+    }
+
+    @Override
+    public void setBackupAvailable(String backupAvailable) {
+        this.backupAvailable = backupAvailable;
+    }
+
+    @Override
+    public void setBackupStorageLimit(String backupStorageLimit) {
+        this.backupStorageLimit = backupStorageLimit;
+    }
+
+    @Override
+    public void setBackupStorageTotal(Long backupStorageTotal) {
+        this.backupStorageTotal = backupStorageTotal;
+    }
+
+    @Override
+    public void setBackupStorageAvailable(String backupStorageAvailable) {
+        this.backupStorageAvailable = backupStorageAvailable;
+    }
+
     @Override
     public void setTemplateLimit(String templateLimit) {
         this.templateLimit = templateLimit;
@@ -537,6 +615,36 @@ public void setSecondaryStorageAvailable(String secondaryStorageAvailable) {
         this.secondaryStorageAvailable = secondaryStorageAvailable;
     }
 
+    @Override
+    public void setBucketLimit(String bucketLimit) {
+        this.bucketLimit = bucketLimit;
+    }
+
+    @Override
+    public void setBucketTotal(Long bucketTotal) {
+        this.bucketTotal = bucketTotal;
+    }
+
+    @Override
+    public void setBucketAvailable(String bucketAvailable) {
+        this.bucketAvailable = bucketAvailable;
+    }
+
+    @Override
+    public void setObjectStorageLimit(String objectStorageLimit) {
+        this.objectStorageLimit = objectStorageLimit;
+    }
+
+    @Override
+    public void setObjectStorageTotal(Long objectStorageTotal) {
+        this.objectStorageTotal = objectStorageTotal;
+    }
+
+    @Override
+    public void setObjectStorageAvailable(String objectStorageAvailable) {
+        this.objectStorageAvailable = objectStorageAvailable;
+    }
+
     public void setDefaultZone(String defaultZoneId) {
         this.defaultZoneId = defaultZoneId;
     }

api/src/main/java/org/apache/cloudstack/api/response/BackupScheduleResponse.java

@@ -37,18 +37,22 @@ public class BackupScheduleResponse extends BaseResponse {
     @Param(description = "ID of the VM")
     private String vmId;
 
-    @SerializedName("schedule")
+    @SerializedName(ApiConstants.SCHEDULE)
     @Param(description = "time the backup is scheduled to be taken.")
     private String schedule;
 
-    @SerializedName("intervaltype")
+    @SerializedName(ApiConstants.INTERVAL_TYPE)
     @Param(description = "the interval type of the backup schedule")
     private DateUtil.IntervalType intervalType;
 
-    @SerializedName("timezone")
+    @SerializedName(ApiConstants.TIMEZONE)
     @Param(description = "the time zone of the backup schedule")
     private String timezone;
 
+    @SerializedName(ApiConstants.MAX_BACKUPS)
+    @Param(description = "maximum number of backups retained")
+    private Integer maxBakups;
+
     public String getVmName() {
         return vmName;
     }
@@ -88,4 +92,8 @@ public String getTimezone() {
     public void setTimezone(String timezone) {
         this.timezone = timezone;
     }
+
+    public void setMaxBakups(Integer maxBakups) {
+        this.maxBakups = maxBakups;
+    }
 }

api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java

@@ -75,7 +75,7 @@ public class BucketResponse extends BaseResponseWithTagInformation implements Co
     private String state;
 
     @SerializedName(ApiConstants.QUOTA)
-    @Param(description = "Bucket Quota in GB")
+    @Param(description = "Bucket Quota in GiB")
     private Integer quota;
 
     @SerializedName(ApiConstants.ENCRYPTION)