Skip to content

Commit

Permalink
chore: improve+streamline logging in Azure Blob test components (#194)
Browse files Browse the repository at this point in the history 
* pr-fix: correct merge w/ 'main'

* chore: improve+streamline blob logging

* pr-fix: faster garbage-collection

* pr-fix: improve garbage-collection
  • Loading branch information
@stijnmoreels
stijnmoreels authored Oct 17, 2024
1 parent fe814b2 commit 63f7b1e
Show file tree
Hide file tree
Showing 2 changed files with 47 additions and 94 deletions.
89 changes: 28 additions & 61 deletions src/Arcus.Testing.Storage.Blob/TemporaryBlobContainer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ public class BlobNameFilter

private BlobNameFilter(Func<string, bool> isMatch)
{
_isMatch = isMatch ?? throw new ArgumentNullException(nameof(isMatch));
ArgumentNullException.ThrowIfNull(isMatch);
_isMatch = isMatch;
}

/// <summary>
Expand Down Expand Up @@ -191,10 +192,7 @@ public OnSetupBlobContainerOptions CleanAllBlobs()
/// <exception cref="ArgumentException">Thrown when any of the <paramref name="filters"/> is <c>null</c>.</exception>>
public OnSetupBlobContainerOptions CleanMatchingBlobs(params BlobNameFilter[] filters)
{
if (filters is null)
{
throw new ArgumentNullException(nameof(filters));
}
ArgumentNullException.ThrowIfNull(filters);

if (Array.Exists(filters, f => f is null))
{
Expand Down Expand Up @@ -224,16 +222,13 @@ public OnSetupBlobContainerOptions LeaveAllBlobs()
/// <exception cref="ArgumentNullException">Thrown when the <paramref name="blob"/> is <c>null</c>.</exception>
internal bool IsMatched(BlobItem blob)
{
if (blob is null)
{
throw new ArgumentNullException(nameof(blob));
}
ArgumentNullException.ThrowIfNull(blob);

return Blobs switch
{
OnSetupContainer.LeaveExisted => false,
OnSetupContainer.CleanIfExisted => true,
OnSetupContainer.CleanIfMatched => _filters.Any(filter => filter.IsMatch(blob)),
OnSetupContainer.CleanIfMatched => _filters.Exists(filter => filter.IsMatch(blob)),
_ => false
};
}
Expand Down Expand Up @@ -288,10 +283,7 @@ public OnTeardownBlobContainerOptions CleanAllBlobs()
/// <exception cref="ArgumentException">Thrown when any of the <paramref name="filters"/> is <c>null</c>.</exception>
public OnTeardownBlobContainerOptions CleanMatchingBlobs(params BlobNameFilter[] filters)
{
if (filters is null)
{
throw new ArgumentNullException(nameof(filters));
}
ArgumentNullException.ThrowIfNull(filters);

if (Array.Exists(filters, f => f is null))
{
Expand Down Expand Up @@ -329,15 +321,12 @@ public OnTeardownBlobContainerOptions DeleteExistingContainer()
/// <exception cref="ArgumentNullException">Thrown when the <paramref name="blob"/> is <c>null</c>.</exception>
internal bool IsMatched(BlobItem blob)
{
if (blob is null)
{
throw new ArgumentNullException(nameof(blob));
}
ArgumentNullException.ThrowIfNull(blob);

return Blobs switch
{
OnTeardownBlobs.CleanAll => true,
OnTeardownBlobs.CleanIfMatched => _filters.Any(filter => filter.IsMatch(blob)),
OnTeardownBlobs.CleanIfMatched => _filters.Exists(filter => filter.IsMatch(blob)),
_ => false
};
}
Expand Down Expand Up @@ -375,11 +364,14 @@ private TemporaryBlobContainer(
TemporaryBlobContainerOptions options,
ILogger logger)
{
ArgumentNullException.ThrowIfNull(containerClient);
ArgumentNullException.ThrowIfNull(options);

_createdByUs = createdByUs;
_options = options ?? new TemporaryBlobContainerOptions();
_options = options;
_logger = logger ?? NullLogger.Instance;

Client = containerClient ?? throw new ArgumentNullException(nameof(containerClient));
Client = containerClient;
}

/// <summary>
Expand All @@ -405,22 +397,8 @@ private TemporaryBlobContainer(
/// <param name="logger">The logger to write diagnostic messages during the lifetime of the Azure Blob container.</param>
/// <exception cref="ArgumentException">Thrown when the <paramref name="accountName"/> or <paramref name="containerName"/> is blank.</exception>
public static async Task<TemporaryBlobContainer> CreateIfNotExistsAsync(string accountName, string containerName, ILogger logger)
{
if (string.IsNullOrWhiteSpace(accountName))
{
throw new ArgumentException(
"Requires a non-blank Azure Storage account name to create a temporary Azure Blob container test fixture," +
" used in container URI: 'https://{account_name}.blob.core.windows.net/{container_name}'", nameof(accountName));
}

if (string.IsNullOrWhiteSpace(containerName))
{
throw new ArgumentException(
"Requires a non-blank Azure Blob container name to create a temporary Azure Blob container test fixture," +
" used in container URI: 'https://{account_name}.blob.core.windows.net/{container_name}'", nameof(containerName));
}

return await CreateIfNotExistsAsync(accountName, containerName, logger ?? NullLogger.Instance, configureOptions: null);
{
return await CreateIfNotExistsAsync(accountName, containerName, logger, configureOptions: null);
}

/// <summary>
Expand Down Expand Up @@ -454,7 +432,7 @@ public static async Task<TemporaryBlobContainer> CreateIfNotExistsAsync(
var blobContainerUri = new Uri($"https://{accountName}.blob.core.windows.net/{containerName}");
var containerClient = new BlobContainerClient(blobContainerUri, new DefaultAzureCredential());

return await CreateIfNotExistsAsync(containerClient, logger ?? NullLogger.Instance, configureOptions);
return await CreateIfNotExistsAsync(containerClient, logger, configureOptions);
}

/// <summary>
Expand All @@ -465,10 +443,7 @@ public static async Task<TemporaryBlobContainer> CreateIfNotExistsAsync(
/// <exception cref="ArgumentNullException">Thrown when the <paramref name="containerClient"/> is <c>null</c>.</exception>
public static async Task<TemporaryBlobContainer> CreateIfNotExistsAsync(BlobContainerClient containerClient, ILogger logger)
{
return await CreateIfNotExistsAsync(
containerClient ?? throw new ArgumentNullException(nameof(containerClient)),
logger,
configureOptions: null);
return await CreateIfNotExistsAsync(containerClient, logger, configureOptions: null);
}

/// <summary>
Expand All @@ -483,18 +458,13 @@ public static async Task<TemporaryBlobContainer> CreateIfNotExistsAsync(
ILogger logger,
Action<TemporaryBlobContainerOptions> configureOptions)
{
if (containerClient is null)
{
throw new ArgumentNullException(nameof(containerClient));
}

ArgumentNullException.ThrowIfNull(containerClient);
logger ??= NullLogger.Instance;

bool createdByUs = await EnsureContainerCreatedAsync(containerClient, logger);

var options = new TemporaryBlobContainerOptions();
configureOptions?.Invoke(options);

bool createdByUs = await EnsureContainerCreatedAsync(containerClient, logger);
await CleanBlobContainerUponCreationAsync(containerClient, options, logger);

return new TemporaryBlobContainer(containerClient, createdByUs, options, logger);
Expand All @@ -505,13 +475,13 @@ private static async Task<bool> EnsureContainerCreatedAsync(BlobContainerClient
bool createdByUs = false;
if (!await containerClient.ExistsAsync())
{
logger.LogDebug("Creating Azure Blob container '{ContainerName}'", containerClient.Name);
logger.LogDebug("[Test:Setup] Create new Azure Blob container '{ContainerName}' in account '{AccountName}'", containerClient.Name, containerClient.AccountName);
await containerClient.CreateIfNotExistsAsync();
createdByUs = true;
}
else
{
logger.LogDebug("Azure Blob container '{ContainerName}' already exists", containerClient.Name);
logger.LogDebug("[Test:Setup] Use already existing Azure Blob container '{ContainerName}' in account '{AccountName}'", containerClient.Name, containerClient.AccountName);
}

return createdByUs;
Expand Down Expand Up @@ -539,16 +509,13 @@ public async Task<BlobClient> UploadBlobAsync(string blobName, BinaryData blobCo
/// <exception cref="ArgumentNullException">Thrown when the <paramref name="blobContent"/> is <c>null</c>.</exception>
public async Task<BlobClient> UploadBlobAsync(string blobName, BinaryData blobContent, Action<TemporaryBlobFileOptions> configureOptions)
{
ArgumentNullException.ThrowIfNull(blobContent);

if (string.IsNullOrWhiteSpace(blobName))
{
throw new ArgumentException($"Requires a non-blank blob name to upload a temporary blob in the temporary '{Name}' container", nameof(blobName));
}

if (blobContent is null)
{
throw new ArgumentNullException(nameof(blobContent));
}

BlobClient blobClient = Client.GetBlobClient(blobName);
_blobs.Add(await TemporaryBlobFile.UploadIfNotExistsAsync(blobClient, blobContent, _logger, configureOptions));

Expand All @@ -573,10 +540,12 @@ public async ValueTask DisposeAsync()
{
disposables.Add(AsyncDisposable.Create(async () =>
{
_logger.LogTrace("Deleting Azure Blob container '{ContainerName}'", Client.Name);
_logger.LogDebug("[Test:Teardown] Delete Azure Blob container '{ContainerName}' from account '{AccountName}'", Client.Name, Client.AccountName);
await Client.DeleteIfExistsAsync();
}));
}

GC.SuppressFinalize(this);
}

private static async Task CleanBlobContainerUponCreationAsync(BlobContainerClient containerClient, TemporaryBlobContainerOptions options, ILogger logger)
Expand All @@ -586,12 +555,11 @@ private static async Task CleanBlobContainerUponCreationAsync(BlobContainerClien
return;
}

logger.LogTrace("Cleaning Azure Blob container '{ContainerName}'", containerClient.Name);
await foreach (BlobItem blob in containerClient.GetBlobsAsync())
{
if (options.OnSetup.IsMatched(blob))
{
logger.LogTrace("Removing blob '{BlobName}' from Azure Blob container '{ContainerName}'", blob.Name, containerClient.Name);
logger.LogTrace("[Test:Setup] Delete Azure Blob file '{BlobName}' from container '{AccountName}/{ContainerName}'", blob.Name, containerClient.AccountName, containerClient.Name);
await containerClient.GetBlobClient(blob.Name).DeleteIfExistsAsync();
}
}
Expand All @@ -604,12 +572,11 @@ private static async Task CleanBlobContainerUponDeletionAsync(BlobContainerClien
return;
}

logger.LogTrace("Cleaning Azure Blob container '{ContainerName}'", containerClient.Name);
await foreach (BlobItem blob in containerClient.GetBlobsAsync())
{
if (options.OnTeardown.IsMatched(blob))
{
logger.LogTrace("Removing blob '{BlobName}' from Azure Blob container '{ContainerName}'", blob.Name, containerClient.Name);
logger.LogTrace("[Test:Teardown] Delete Azure Blob file '{BlobName}' from container '{AccountName}/{ContainerName}'", blob.Name, containerClient.AccountName, containerClient.Name);
await containerClient.GetBlobClient(blob.Name).DeleteIfExistsAsync();
}
}
Expand Down
52 changes: 19 additions & 33 deletions src/Arcus.Testing.Storage.Blob/TemporaryBlobFile.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,12 +109,15 @@ private TemporaryBlobFile(
TemporaryBlobFileOptions options,
ILogger logger)
{
ArgumentNullException.ThrowIfNull(blobClient);
ArgumentNullException.ThrowIfNull(options);

_createdByUs = createdByUs;
_originalData = originalData;
_options = options;
_logger = logger ?? NullLogger.Instance;

Client = blobClient ?? throw new ArgumentNullException(nameof(blobClient));
Client = blobClient;
}

/// <summary>
Expand Down Expand Up @@ -160,9 +163,9 @@ public static async Task<TemporaryBlobFile> UploadIfNotExistsAsync(Uri blobConta
}

return await UploadIfNotExistsAsync(
blobContainerUri ?? throw new ArgumentNullException(nameof(blobContainerUri)),
blobContainerUri,
blobName,
blobContent ?? throw new ArgumentNullException(nameof(blobContent)),
blobContent,
logger,
configureOptions: null);
}
Expand All @@ -181,7 +184,7 @@ public static async Task<TemporaryBlobFile> UploadIfNotExistsAsync(Uri blobConta
/// <param name="blobContent">The content of the blob to upload.</param>
/// <param name="logger">The logger to write diagnostic messages during the upload process.</param>
/// <param name="configureOptions">The function to configure the additional options of how the blob should be uploaded.</param>
/// <exception cref="ArgumentException">Thrown when the <paramref name="blobContainerUri"/> or the <paramref name="blobName"/> is blank.</exception>
/// <exception cref="ArgumentException">Thrown when the <paramref name="blobName"/> is blank.</exception>
/// <exception cref="ArgumentNullException">Thrown when <paramref name="blobContainerUri"/> or the <paramref name="blobContent"/> is <c>null</c>.</exception>
public static async Task<TemporaryBlobFile> UploadIfNotExistsAsync(
Uri blobContainerUri,
Expand All @@ -190,21 +193,14 @@ public static async Task<TemporaryBlobFile> UploadIfNotExistsAsync(
ILogger logger,
Action<TemporaryBlobFileOptions> configureOptions)
{
if (blobContainerUri is null)
{
throw new ArgumentNullException(nameof(blobContainerUri));
}
ArgumentNullException.ThrowIfNull(blobContainerUri);
ArgumentNullException.ThrowIfNull(blobContent);

if (string.IsNullOrWhiteSpace(blobName))
{
throw new ArgumentException("Requires a non-blank name for the Azure Blob file name for it to be uploaded to Azure Blob storage", nameof(blobName));
}

if (blobContent is null)
{
throw new ArgumentNullException(nameof(blobContent));
}

var containerClient = new BlobContainerClient(blobContainerUri, new DefaultAzureCredential());
BlobClient blobClient = containerClient.GetBlobClient(blobName);

Expand All @@ -220,11 +216,7 @@ public static async Task<TemporaryBlobFile> UploadIfNotExistsAsync(
/// <exception cref="ArgumentNullException">Thrown when the <paramref name="blobClient"/> or the <paramref name="blobContent"/> is <c>null</c>.</exception>
public static async Task<TemporaryBlobFile> UploadIfNotExistsAsync(BlobClient blobClient, BinaryData blobContent, ILogger logger)
{
return await UploadIfNotExistsAsync(
blobClient ?? throw new ArgumentNullException(nameof(blobClient)),
blobContent ?? throw new ArgumentNullException(nameof(blobContent)),
logger,
configureOptions: null);
return await UploadIfNotExistsAsync(blobClient, blobContent, logger, configureOptions: null);
}

/// <summary>
Expand All @@ -241,17 +233,10 @@ public static async Task<TemporaryBlobFile> UploadIfNotExistsAsync(
ILogger logger,
Action<TemporaryBlobFileOptions> configureOptions)
{
if (blobClient is null)
{
throw new ArgumentNullException(nameof(blobClient));
}

if (blobContent is null)
{
throw new ArgumentNullException(nameof(blobContent));
}

ArgumentNullException.ThrowIfNull(blobClient);
ArgumentNullException.ThrowIfNull(blobContent);
logger ??= NullLogger.Instance;

var options = new TemporaryBlobFileOptions();
configureOptions?.Invoke(options);

Expand All @@ -268,19 +253,18 @@ public static async Task<TemporaryBlobFile> UploadIfNotExistsAsync(
{
if (await client.ExistsAsync())
{
logger.LogDebug("Azure Blob '{BlobName}' already exists in container '{ContainerName}'", client.Name, client.BlobContainerName);
BlobDownloadResult originalContent = await client.DownloadContentAsync();

if (options.OnSetup.OverrideBlob)
{
logger.LogDebug("Override existing Azure Blob '{BlobName}' in container '{ContainerName}'", client.Name, client.BlobContainerName);
logger.LogDebug("[Test:Setup] Replace already existing Azure Blob file '{BlobName}' in container '{AccountName}/{ContainerName}'", client.Name, client.AccountName, client.BlobContainerName);
await client.UploadAsync(newContent, overwrite: true);
}

return (createdByUs: false, originalContent.Content);
}

logger.LogTrace("Uploading Azure Blob '{BlobName}' to container '{ContainerName}'", client.Name, client.BlobContainerName);
logger.LogDebug("[Test:Setup] Upload Azure Blob file '{BlobName}' to container '{AccountName}/{ContainerName}'", client.Name, client.AccountName, client.BlobContainerName);
await client.UploadAsync(newContent);

return (createdByUs: true, originalData: null);
Expand All @@ -294,15 +278,17 @@ public async ValueTask DisposeAsync()
{
if (!_createdByUs && _originalData != null && _options.OnTeardown.Content != OnTeardownBlob.DeleteIfExisted)
{
_logger.LogDebug("Reverting Azure Blob '{BlobName}' original content in container '{ContainerName}'", Client.Name, Client.BlobContainerName);
_logger.LogDebug("[Test:Teardown] Revert replaced Azure Blob file '{BlobName}' to original content in container '{AccountName}/{ContainerName}'", Client.Name, Client.AccountName, Client.BlobContainerName);
await Client.UploadAsync(_originalData, overwrite: true);
}

if (_createdByUs || _options.OnTeardown.Content is OnTeardownBlob.DeleteIfExisted)
{
_logger.LogTrace("Deleting Azure Blob '{BlobName}' from container '{ContainerName}'", Client.Name, Client.BlobContainerName);
_logger.LogDebug("[Test:Teardown] Delete Azure Blob file '{BlobName}' from container '{AccountName}/{ContainerName}'", Client.Name, Client.AccountName, Client.BlobContainerName);
await Client.DeleteIfExistsAsync();
}

GC.SuppressFinalize(this);
}
}
}

0 comments on commit 63f7b1e

Please sign in to comment.