diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj b/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj index 37d1d0d7fa..c079e705e4 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/BuildingBlocks.Infrastructure.csproj @@ -2,6 +2,7 @@ + diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs index a5fda77ee6..2da410b940 100644 --- a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/BlobStorageServiceCollectionExtensions.cs @@ -2,6 +2,7 @@ using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.AzureStorageAccount; using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.GoogleCloudStorage; +using Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.S3; using Backbone.Tooling.Extensions; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; @@ -28,6 +29,10 @@ public static void AddBlobStorage(this IServiceCollection services, BlobStorageO case BlobStorageOptions.GOOGLE_CLOUD_STORAGE: services.AddGoogleCloudStorage(options.GoogleCloudStorage!); break; + case BlobStorageOptions.S3_BUCKET: + services.AddS3(options.S3Bucket!); + break; + default: { if (options.ProductName.IsNullOrEmpty()) @@ -52,18 +57,22 @@ public class BlobStorageOptions : IValidatableObject { public const string AZURE_STORAGE_ACCOUNT = "AzureStorageAccount"; public const string GOOGLE_CLOUD_STORAGE = "GoogleCloudStorage"; + public const string S3_BUCKET = "S3Bucket"; - [RegularExpression($"{AZURE_STORAGE_ACCOUNT}|{GOOGLE_CLOUD_STORAGE}")] + [RegularExpression($"{AZURE_STORAGE_ACCOUNT}|{GOOGLE_CLOUD_STORAGE}|{S3_BUCKET}")] public string ProductName { get; set; } = null!; public AzureStorageAccountOptions? AzureStorageAccount { get; set; } public GoogleCloudStorageOptions? GoogleCloudStorage { get; set; } + public S3BucketOptions? S3Bucket { get; set; } + public string RootFolder => ProductName switch { AZURE_STORAGE_ACCOUNT => AzureStorageAccount!.ContainerName, GOOGLE_CLOUD_STORAGE => GoogleCloudStorage!.BucketName, + S3_BUCKET => S3Bucket!.BucketName, _ => throw new Exception("Unsupported ProductName") }; diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3BlobStorage.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3BlobStorage.cs new file mode 100644 index 0000000000..b4e4a74281 --- /dev/null +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3BlobStorage.cs @@ -0,0 +1,217 @@ +using System.Net; +using Amazon.S3; +using Amazon.S3.Model; +using Amazon.S3.Transfer; +using Backbone.BuildingBlocks.Application.Abstractions.Exceptions; +using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.S3; + +public class S3BlobStorage : IBlobStorage, IDisposable +{ + private readonly AmazonS3Client _s3Client; + private readonly List _changedBlobs; + private readonly IList _removedBlobs; + private readonly string _bucketName; + private readonly ILogger _logger; + + public S3BlobStorage(IOptions config, ILogger logger) + { + var s3Config = new AmazonS3Config + { + ServiceURL = config.Value.ServiceUrl, + ForcePathStyle = true + }; + + _s3Client = new AmazonS3Client(config.Value.AccessKeyId, config.Value.SecretAccessKey, s3Config); + _changedBlobs = []; + _removedBlobs = []; + _bucketName = config.Value.BucketName; + _logger = logger; + } + + public void Add(string folder, string id, byte[] content) + { + _changedBlobs.Add(new ChangedBlob(folder, id, content)); + } + + public void Remove(string folder, string id) + { + _removedBlobs.Add(new RemovedBlob(folder, id)); + } + + public void Dispose() + { + _changedBlobs.Clear(); + _removedBlobs.Clear(); + } + + public async Task FindAsync(string folder, string id) + { + _logger.LogTrace("Reading blob with key '{blobId}'...", id); + + try + { + var request = new GetObjectRequest + { + BucketName = _bucketName, + Key = $"{folder}/{id}" + }; + + using var response = await _s3Client.GetObjectAsync(request); + using var memoryStream = new MemoryStream(); + await response.ResponseStream.CopyToAsync(memoryStream); + + _logger.LogTrace("Found blob with key '{blobId}'.", id); + return memoryStream.ToArray(); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + _logger.LogError("A blob with key '{blobId}' was not found.", id); + throw new NotFoundException("Blob", ex); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error downloading blob with key '{blobId}'.", id); + throw; + } + } + + public Task> FindAllAsync(string folder, string? prefix = null) + { + return Task.FromResult(FindAllBlobsAsync(folder, prefix)); + } + + private async IAsyncEnumerable FindAllBlobsAsync(string folder, string? prefix) + { + _logger.LogTrace("Listing all blobs..."); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = prefix != null ? $"{folder}/{prefix}" : folder + }; + + ListObjectsV2Response response; + do + { + response = await _s3Client.ListObjectsV2Async(request); + + foreach (var obj in response.S3Objects) + { + yield return obj.Key; + } + + request.ContinuationToken = response.NextContinuationToken; + } while (response.IsTruncated); + + _logger.LogTrace("Found all blobs."); + } + + public async Task SaveAsync() + { + await UploadChangedBlobs(); + await DeleteRemovedBlobs(); + } + + private async Task UploadChangedBlobs() + { + _logger.LogTrace("Uploading '{changedBlobsCount}' changed blobs...", _changedBlobs.Count); + + var changedBlobs = new List(_changedBlobs); + + foreach (var blob in changedBlobs) + { + await EnsureKeyDoesNotExist(blob.Folder, blob.Name); + + using var memoryStream = new MemoryStream(blob.Content); + + try + { + _logger.LogTrace("Uploading blob with key '{blobName}'...", blob.Name); + + var request = new TransferUtilityUploadRequest + { + InputStream = memoryStream, + Key = $"{blob.Folder}/{blob.Name}", + BucketName = _bucketName + }; + + var transferUtility = new TransferUtility(_s3Client); + await transferUtility.UploadAsync(request); + + _logger.LogTrace("Upload of blob with key '{blobName}' was successful.", blob.Name); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error uploading blob with key '{blobName}'.", blob.Name); + throw; + } + finally + { + _changedBlobs.Remove(blob); + } + } + } + + private async Task EnsureKeyDoesNotExist(string folder, string key) + { + try + { + var request = new GetObjectRequest + { + BucketName = _bucketName, + Key = $"{folder}/{key}" + }; + + await _s3Client.GetObjectAsync(request); + + _logger.LogError("A blob with key '{blobName}' already exists.", key); + throw new BlobAlreadyExistsException(key); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + } + } + + private async Task DeleteRemovedBlobs() + { + _logger.LogTrace("Deleting '{removedBlobsCount}' blobs...", _removedBlobs.Count); + + var blobsToDelete = new List(_removedBlobs); + + foreach (var blob in blobsToDelete) + { + try + { + var request = new DeleteObjectRequest + { + BucketName = _bucketName, + Key = $"{blob.Folder}/{blob.Name}" + }; + + await _s3Client.DeleteObjectAsync(request); + + _removedBlobs.Remove(blob); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + _logger.LogError("A blob with key '{blobId}' was not found.", blob.Name); + throw new NotFoundException($"Blob with key '{blob.Name}' was not found.", ex); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error deleting blob with key '{blobName}'.", blob.Name); + throw; + } + } + + _logger.LogTrace("Deletion successful."); + } + + private record ChangedBlob(string Folder, string Name, byte[] Content); + + private record RemovedBlob(string Folder, string Name); +} diff --git a/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ServiceCollectionExtensions.cs b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ServiceCollectionExtensions.cs new file mode 100644 index 0000000000..ea537db405 --- /dev/null +++ b/BuildingBlocks/src/BuildingBlocks.Infrastructure/Persistence/BlobStorage/S3/S3ServiceCollectionExtensions.cs @@ -0,0 +1,45 @@ +using System.ComponentModel.DataAnnotations; +using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage; +using Microsoft.Extensions.DependencyInjection; + +namespace Backbone.BuildingBlocks.Infrastructure.Persistence.BlobStorage.S3; + +public static class S3ServiceCollectionExtensions +{ + public static void AddS3(this IServiceCollection services, + Action setupOptions) + { + var options = new S3BucketOptions(); + setupOptions.Invoke(options); + + services.AddS3(options); + } + + public static void AddS3(this IServiceCollection services, S3BucketOptions options) + { + services.Configure(s3Options => + { + s3Options.BucketName = options.BucketName; + s3Options.AccessKeyId = options.AccessKeyId; + s3Options.SecretAccessKey = options.SecretAccessKey; + s3Options.ServiceUrl = options.ServiceUrl; + }); + + services.AddScoped(); + } +} + +public class S3BucketOptions +{ + [Required] + public string ServiceUrl { get; set; } = string.Empty; + + [Required] + public string AccessKeyId { get; set; } = string.Empty; + + [Required] + public string SecretAccessKey { get; set; } = string.Empty; + + [Required] + public string BucketName { get; set; } = string.Empty; +} diff --git a/helm/values.yaml b/helm/values.yaml index c6ed6a4d43..38dbd96684 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -680,6 +680,16 @@ global: # serviceAccountJson: "" # bucketName - the name of the bucket that should be used to store the files # bucketName: "" + # s3Bucket - only applicable if ProductName is "S3Bucket" + # s3Bucket: + # accessKeyId - the access key id that should be used to authenticate + # accessKeyId: "" + # secretAccessKey - the secret access key that should be used to authenticate + # secretAccessKey: "" + # bucketName - the name of the bucket that should be used to store the files + # bucketName: "" + # serviceUrl - the url of the S3 service + # serviceUrl: "" messages: application: # didDomainName - the didDomainName that should be used when generating Identity Addresses