Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: improve+streamline logging in Azure Blob test components #194

Merged
merged 10 commits into from
Oct 17, 2024
Merged

chore: improve+streamline logging in Azure Blob test components #194

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
c07d3e9
pr-fix: correct merge w/ 'main'
stijnmoreels Aug 2, 2024
6b26d6d
Merge branch 'main' of https://github.com/stijnmoreels/arcus.testing
stijnmoreels Aug 2, 2024
0de9e56
Merge remote-tracking branch 'upstream/main'
stijnmoreels Aug 26, 2024
cbfd3d9
Merge remote-tracking branch 'upstream/main'
stijnmoreels Sep 6, 2024
cc62c4b
Merge remote-tracking branch 'upstream/main'
stijnmoreels Sep 13, 2024
7ad43ef
Merge remote-tracking branch 'upstream/main'
stijnmoreels Sep 23, 2024
483f449
Merge remote-tracking branch 'upstream/main'
stijnmoreels Sep 27, 2024
6e3fced
chore: improve+streamline blob logging
stijnmoreels Oct 2, 2024
965ea0e
pr-fix: faster garbage-collection
stijnmoreels Oct 2, 2024
7ee4e22
pr-fix: improve garbage-collection
stijnmoreels Oct 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 28 additions & 61 deletions src/Arcus.Testing.Storage.Blob/TemporaryBlobContainer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ public class BlobNameFilter

private BlobNameFilter(Func<string, bool> isMatch)
{
_isMatch = isMatch ?? throw new ArgumentNullException(nameof(isMatch));
ArgumentNullException.ThrowIfNull(isMatch);
_isMatch = isMatch;
}

/// <summary>
Expand Down Expand Up @@ -191,10 +192,7 @@ public OnSetupBlobContainerOptions CleanAllBlobs()
/// <exception cref="ArgumentException">Thrown when any of the <paramref name="filters"/> is <c>null</c>.</exception>>
public OnSetupBlobContainerOptions CleanMatchingBlobs(params BlobNameFilter[] filters)
{
if (filters is null)
{
throw new ArgumentNullException(nameof(filters));
}
ArgumentNullException.ThrowIfNull(filters);

if (Array.Exists(filters, f => f is null))
{
Expand Down Expand Up @@ -224,16 +222,13 @@ public OnSetupBlobContainerOptions LeaveAllBlobs()
/// <exception cref="ArgumentNullException">Thrown when the <paramref name="blob"/> is <c>null</c>.</exception>
internal bool IsMatched(BlobItem blob)
{
if (blob is null)
{
throw new ArgumentNullException(nameof(blob));
}
ArgumentNullException.ThrowIfNull(blob);

return Blobs switch
{
OnSetupContainer.LeaveExisted => false,
OnSetupContainer.CleanIfExisted => true,
OnSetupContainer.CleanIfMatched => _filters.Any(filter => filter.IsMatch(blob)),
OnSetupContainer.CleanIfMatched => _filters.Exists(filter => filter.IsMatch(blob)),
_ => false
};
}
Expand Down Expand Up @@ -288,10 +283,7 @@ public OnTeardownBlobContainerOptions CleanAllBlobs()
/// <exception cref="ArgumentException">Thrown when any of the <paramref name="filters"/> is <c>null</c>.</exception>
public OnTeardownBlobContainerOptions CleanMatchingBlobs(params BlobNameFilter[] filters)
{
if (filters is null)
{
throw new ArgumentNullException(nameof(filters));
}
ArgumentNullException.ThrowIfNull(filters);

if (Array.Exists(filters, f => f is null))
{
Expand Down Expand Up @@ -329,15 +321,12 @@ public OnTeardownBlobContainerOptions DeleteExistingContainer()
/// <exception cref="ArgumentNullException">Thrown when the <paramref name="blob"/> is <c>null</c>.</exception>
internal bool IsMatched(BlobItem blob)
{
if (blob is null)
{
throw new ArgumentNullException(nameof(blob));
}
ArgumentNullException.ThrowIfNull(blob);

return Blobs switch
{
OnTeardownBlobs.CleanAll => true,
OnTeardownBlobs.CleanIfMatched => _filters.Any(filter => filter.IsMatch(blob)),
OnTeardownBlobs.CleanIfMatched => _filters.Exists(filter => filter.IsMatch(blob)),
_ => false
};
}
Expand Down Expand Up @@ -375,11 +364,14 @@ private TemporaryBlobContainer(
TemporaryBlobContainerOptions options,
ILogger logger)
{
ArgumentNullException.ThrowIfNull(containerClient);
ArgumentNullException.ThrowIfNull(options);

_createdByUs = createdByUs;
_options = options ?? new TemporaryBlobContainerOptions();
_options = options;
_logger = logger ?? NullLogger.Instance;

Client = containerClient ?? throw new ArgumentNullException(nameof(containerClient));
Client = containerClient;
}

/// <summary>
Expand All @@ -405,22 +397,8 @@ private TemporaryBlobContainer(
/// <param name="logger">The logger to write diagnostic messages during the lifetime of the Azure Blob container.</param>
/// <exception cref="ArgumentException">Thrown when the <paramref name="accountName"/> or <paramref name="containerName"/> is blank.</exception>
public static async Task<TemporaryBlobContainer> CreateIfNotExistsAsync(string accountName, string containerName, ILogger logger)
{
if (string.IsNullOrWhiteSpace(accountName))
{
throw new ArgumentException(
"Requires a non-blank Azure Storage account name to create a temporary Azure Blob container test fixture," +
" used in container URI: 'https://{account_name}.blob.core.windows.net/{container_name}'", nameof(accountName));
}

if (string.IsNullOrWhiteSpace(containerName))
{
throw new ArgumentException(
"Requires a non-blank Azure Blob container name to create a temporary Azure Blob container test fixture," +
" used in container URI: 'https://{account_name}.blob.core.windows.net/{container_name}'", nameof(containerName));
}

return await CreateIfNotExistsAsync(accountName, containerName, logger ?? NullLogger.Instance, configureOptions: null);
{
return await CreateIfNotExistsAsync(accountName, containerName, logger, configureOptions: null);
}

/// <summary>
Expand Down Expand Up @@ -454,7 +432,7 @@ public static async Task<TemporaryBlobContainer> CreateIfNotExistsAsync(
var blobContainerUri = new Uri($"https://{accountName}.blob.core.windows.net/{containerName}");
var containerClient = new BlobContainerClient(blobContainerUri, new DefaultAzureCredential());

return await CreateIfNotExistsAsync(containerClient, logger ?? NullLogger.Instance, configureOptions);
return await CreateIfNotExistsAsync(containerClient, logger, configureOptions);
}

/// <summary>
Expand All @@ -465,10 +443,7 @@ public static async Task<TemporaryBlobContainer> CreateIfNotExistsAsync(
/// <exception cref="ArgumentNullException">Thrown when the <paramref name="containerClient"/> is <c>null</c>.</exception>
public static async Task<TemporaryBlobContainer> CreateIfNotExistsAsync(BlobContainerClient containerClient, ILogger logger)
{
return await CreateIfNotExistsAsync(
containerClient ?? throw new ArgumentNullException(nameof(containerClient)),
logger,
configureOptions: null);
return await CreateIfNotExistsAsync(containerClient, logger, configureOptions: null);
}

/// <summary>
Expand All @@ -483,18 +458,13 @@ public static async Task<TemporaryBlobContainer> CreateIfNotExistsAsync(
ILogger logger,
Action<TemporaryBlobContainerOptions> configureOptions)
{
if (containerClient is null)
{
throw new ArgumentNullException(nameof(containerClient));
}

ArgumentNullException.ThrowIfNull(containerClient);
logger ??= NullLogger.Instance;

bool createdByUs = await EnsureContainerCreatedAsync(containerClient, logger);

var options = new TemporaryBlobContainerOptions();
configureOptions?.Invoke(options);

bool createdByUs = await EnsureContainerCreatedAsync(containerClient, logger);
await CleanBlobContainerUponCreationAsync(containerClient, options, logger);

return new TemporaryBlobContainer(containerClient, createdByUs, options, logger);
Expand All @@ -505,13 +475,13 @@ private static async Task<bool> EnsureContainerCreatedAsync(BlobContainerClient
bool createdByUs = false;
if (!await containerClient.ExistsAsync())
{
logger.LogDebug("Creating Azure Blob container '{ContainerName}'", containerClient.Name);
logger.LogDebug("[Test:Setup] Create new Azure Blob container '{ContainerName}' in account '{AccountName}'", containerClient.Name, containerClient.AccountName);
await containerClient.CreateIfNotExistsAsync();
createdByUs = true;
}
else
{
logger.LogDebug("Azure Blob container '{ContainerName}' already exists", containerClient.Name);
logger.LogDebug("[Test:Setup] Use already existing Azure Blob container '{ContainerName}' in account '{AccountName}'", containerClient.Name, containerClient.AccountName);
}

return createdByUs;
Expand Down Expand Up @@ -539,16 +509,13 @@ public async Task<BlobClient> UploadBlobAsync(string blobName, BinaryData blobCo
/// <exception cref="ArgumentNullException">Thrown when the <paramref name="blobContent"/> is <c>null</c>.</exception>
public async Task<BlobClient> UploadBlobAsync(string blobName, BinaryData blobContent, Action<TemporaryBlobFileOptions> configureOptions)
{
ArgumentNullException.ThrowIfNull(blobContent);

if (string.IsNullOrWhiteSpace(blobName))
{
throw new ArgumentException($"Requires a non-blank blob name to upload a temporary blob in the temporary '{Name}' container", nameof(blobName));
}

if (blobContent is null)
{
throw new ArgumentNullException(nameof(blobContent));
}

BlobClient blobClient = Client.GetBlobClient(blobName);
_blobs.Add(await TemporaryBlobFile.UploadIfNotExistsAsync(blobClient, blobContent, _logger, configureOptions));

Expand All @@ -573,10 +540,12 @@ public async ValueTask DisposeAsync()
{
disposables.Add(AsyncDisposable.Create(async () =>
{
_logger.LogTrace("Deleting Azure Blob container '{ContainerName}'", Client.Name);
_logger.LogDebug("[Test:Teardown] Delete Azure Blob container '{ContainerName}' from account '{AccountName}'", Client.Name, Client.AccountName);
await Client.DeleteIfExistsAsync();
}));
}

GC.SuppressFinalize(this);
}

private static async Task CleanBlobContainerUponCreationAsync(BlobContainerClient containerClient, TemporaryBlobContainerOptions options, ILogger logger)
Expand All @@ -586,12 +555,11 @@ private static async Task CleanBlobContainerUponCreationAsync(BlobContainerClien
return;
}

logger.LogTrace("Cleaning Azure Blob container '{ContainerName}'", containerClient.Name);
await foreach (BlobItem blob in containerClient.GetBlobsAsync())
{
if (options.OnSetup.IsMatched(blob))
{
logger.LogTrace("Removing blob '{BlobName}' from Azure Blob container '{ContainerName}'", blob.Name, containerClient.Name);
logger.LogTrace("[Test:Setup] Delete Azure Blob file '{BlobName}' from container '{AccountName}/{ContainerName}'", blob.Name, containerClient.AccountName, containerClient.Name);
await containerClient.GetBlobClient(blob.Name).DeleteIfExistsAsync();
}
}
Expand All @@ -604,12 +572,11 @@ private static async Task CleanBlobContainerUponDeletionAsync(BlobContainerClien
return;
}

logger.LogTrace("Cleaning Azure Blob container '{ContainerName}'", containerClient.Name);
await foreach (BlobItem blob in containerClient.GetBlobsAsync())
{
if (options.OnTeardown.IsMatched(blob))
{
logger.LogTrace("Removing blob '{BlobName}' from Azure Blob container '{ContainerName}'", blob.Name, containerClient.Name);
logger.LogTrace("[Test:Teardown] Delete Azure Blob file '{BlobName}' from container '{AccountName}/{ContainerName}'", blob.Name, containerClient.AccountName, containerClient.Name);
await containerClient.GetBlobClient(blob.Name).DeleteIfExistsAsync();
}
}
Expand Down
52 changes: 19 additions & 33 deletions src/Arcus.Testing.Storage.Blob/TemporaryBlobFile.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,12 +109,15 @@ private TemporaryBlobFile(
TemporaryBlobFileOptions options,
ILogger logger)
{
ArgumentNullException.ThrowIfNull(blobClient);
ArgumentNullException.ThrowIfNull(options);

_createdByUs = createdByUs;
_originalData = originalData;
_options = options;
_logger = logger ?? NullLogger.Instance;

Client = blobClient ?? throw new ArgumentNullException(nameof(blobClient));
Client = blobClient;
}

/// <summary>
Expand Down Expand Up @@ -160,9 +163,9 @@ public static async Task<TemporaryBlobFile> UploadIfNotExistsAsync(Uri blobConta
}

return await UploadIfNotExistsAsync(
blobContainerUri ?? throw new ArgumentNullException(nameof(blobContainerUri)),
blobContainerUri,
blobName,
blobContent ?? throw new ArgumentNullException(nameof(blobContent)),
blobContent,
logger,
configureOptions: null);
}
Expand All @@ -181,7 +184,7 @@ public static async Task<TemporaryBlobFile> UploadIfNotExistsAsync(Uri blobConta
/// <param name="blobContent">The content of the blob to upload.</param>
/// <param name="logger">The logger to write diagnostic messages during the upload process.</param>
/// <param name="configureOptions">The function to configure the additional options of how the blob should be uploaded.</param>
/// <exception cref="ArgumentException">Thrown when the <paramref name="blobContainerUri"/> or the <paramref name="blobName"/> is blank.</exception>
/// <exception cref="ArgumentException">Thrown when the <paramref name="blobName"/> is blank.</exception>
/// <exception cref="ArgumentNullException">Thrown when <paramref name="blobContainerUri"/> or the <paramref name="blobContent"/> is <c>null</c>.</exception>
public static async Task<TemporaryBlobFile> UploadIfNotExistsAsync(
Uri blobContainerUri,
Expand All @@ -190,21 +193,14 @@ public static async Task<TemporaryBlobFile> UploadIfNotExistsAsync(
ILogger logger,
Action<TemporaryBlobFileOptions> configureOptions)
{
if (blobContainerUri is null)
{
throw new ArgumentNullException(nameof(blobContainerUri));
}
ArgumentNullException.ThrowIfNull(blobContainerUri);
ArgumentNullException.ThrowIfNull(blobContent);

if (string.IsNullOrWhiteSpace(blobName))
{
throw new ArgumentException("Requires a non-blank name for the Azure Blob file name for it to be uploaded to Azure Blob storage", nameof(blobName));
}

if (blobContent is null)
{
throw new ArgumentNullException(nameof(blobContent));
}

var containerClient = new BlobContainerClient(blobContainerUri, new DefaultAzureCredential());
BlobClient blobClient = containerClient.GetBlobClient(blobName);

Expand All @@ -220,11 +216,7 @@ public static async Task<TemporaryBlobFile> UploadIfNotExistsAsync(
/// <exception cref="ArgumentNullException">Thrown when the <paramref name="blobClient"/> or the <paramref name="blobContent"/> is <c>null</c>.</exception>
public static async Task<TemporaryBlobFile> UploadIfNotExistsAsync(BlobClient blobClient, BinaryData blobContent, ILogger logger)
{
return await UploadIfNotExistsAsync(
blobClient ?? throw new ArgumentNullException(nameof(blobClient)),
blobContent ?? throw new ArgumentNullException(nameof(blobContent)),
logger,
configureOptions: null);
return await UploadIfNotExistsAsync(blobClient, blobContent, logger, configureOptions: null);
}

/// <summary>
Expand All @@ -241,17 +233,10 @@ public static async Task<TemporaryBlobFile> UploadIfNotExistsAsync(
ILogger logger,
Action<TemporaryBlobFileOptions> configureOptions)
{
if (blobClient is null)
{
throw new ArgumentNullException(nameof(blobClient));
}

if (blobContent is null)
{
throw new ArgumentNullException(nameof(blobContent));
}

ArgumentNullException.ThrowIfNull(blobClient);
ArgumentNullException.ThrowIfNull(blobContent);
logger ??= NullLogger.Instance;

var options = new TemporaryBlobFileOptions();
configureOptions?.Invoke(options);

Expand All @@ -268,19 +253,18 @@ public static async Task<TemporaryBlobFile> UploadIfNotExistsAsync(
{
if (await client.ExistsAsync())
{
logger.LogDebug("Azure Blob '{BlobName}' already exists in container '{ContainerName}'", client.Name, client.BlobContainerName);
BlobDownloadResult originalContent = await client.DownloadContentAsync();

if (options.OnSetup.OverrideBlob)
{
logger.LogDebug("Override existing Azure Blob '{BlobName}' in container '{ContainerName}'", client.Name, client.BlobContainerName);
logger.LogDebug("[Test:Setup] Replace already existing Azure Blob file '{BlobName}' in container '{AccountName}/{ContainerName}'", client.Name, client.AccountName, client.BlobContainerName);
await client.UploadAsync(newContent, overwrite: true);
}

return (createdByUs: false, originalContent.Content);
}

logger.LogTrace("Uploading Azure Blob '{BlobName}' to container '{ContainerName}'", client.Name, client.BlobContainerName);
logger.LogDebug("[Test:Setup] Upload Azure Blob file '{BlobName}' to container '{AccountName}/{ContainerName}'", client.Name, client.AccountName, client.BlobContainerName);
await client.UploadAsync(newContent);

return (createdByUs: true, originalData: null);
Expand All @@ -294,15 +278,17 @@ public async ValueTask DisposeAsync()
{
if (!_createdByUs && _originalData != null && _options.OnTeardown.Content != OnTeardownBlob.DeleteIfExisted)
{
_logger.LogDebug("Reverting Azure Blob '{BlobName}' original content in container '{ContainerName}'", Client.Name, Client.BlobContainerName);
_logger.LogDebug("[Test:Teardown] Revert replaced Azure Blob file '{BlobName}' to original content in container '{AccountName}/{ContainerName}'", Client.Name, Client.AccountName, Client.BlobContainerName);
await Client.UploadAsync(_originalData, overwrite: true);
}

if (_createdByUs || _options.OnTeardown.Content is OnTeardownBlob.DeleteIfExisted)
{
_logger.LogTrace("Deleting Azure Blob '{BlobName}' from container '{ContainerName}'", Client.Name, Client.BlobContainerName);
_logger.LogDebug("[Test:Teardown] Delete Azure Blob file '{BlobName}' from container '{AccountName}/{ContainerName}'", Client.Name, Client.AccountName, Client.BlobContainerName);
await Client.DeleteIfExistsAsync();
}

GC.SuppressFinalize(this);
}
}
}
Loading