diff --git a/src/OrasProject.Oras/Registry/Reference.cs b/src/OrasProject.Oras/Registry/Reference.cs index 54eca43..f52ca3a 100644 --- a/src/OrasProject.Oras/Registry/Reference.cs +++ b/src/OrasProject.Oras/Registry/Reference.cs @@ -201,6 +201,18 @@ public static bool TryParse(string reference, [NotNullWhen(true)] out Reference? } } + public Reference(Reference other) + { + if (other == null) + { + throw new ArgumentNullException(nameof(other)); + } + + _registry = other.Registry; + _repository = other.Repository; + ContentReference = other.ContentReference; + } + public Reference(string registry) => _registry = ValidateRegistry(registry); public Reference(string registry, string? repository) : this(registry) diff --git a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs index 58239d9..7c036b7 100644 --- a/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs +++ b/src/OrasProject.Oras/Registry/Remote/HttpResponseMessageExtensions.cs @@ -98,7 +98,7 @@ public static void VerifyContentDigest(this HttpResponseMessage response, string } if (contentDigest != expected) { - throw new HttpIOException(HttpRequestError.InvalidResponse, $"{response.RequestMessage!.Method} {response.RequestMessage.RequestUri}: invalid response; digest mismatch in Docker-Content-Digest: received {contentDigest} when expecting {digestStr}"); + throw new HttpIOException(HttpRequestError.InvalidResponse, $"{response.RequestMessage!.Method} {response.RequestMessage.RequestUri}: invalid response; digest mismatch in Docker-Content-Digest: received {contentDigest} when expecting {expected}"); } } diff --git a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs index b0e35c5..9df2271 100644 --- a/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs +++ b/src/OrasProject.Oras/Registry/Remote/ManifestStore.cs @@ -397,5 +397,79 @@ public async Task TagAsync(Descriptor descriptor, string reference, Cancellation /// /// public async Task DeleteAsync(Descriptor target, CancellationToken cancellationToken = default) - => await Repository.DeleteAsync(target, true, cancellationToken).ConfigureAwait(false); + => await DeleteWithIndexing(target, cancellationToken).ConfigureAwait(false); + + /// + /// DeleteWithIndexing deletes the specified target (Descriptor) from the repository, + /// handling referrer indexing if necessary. + /// + /// The target descriptor to delete. + /// A cancellation token to cancel the operation if needed. Defaults to default. + /// + private async Task DeleteWithIndexing(Descriptor target, CancellationToken cancellationToken = default) + { + switch (target.MediaType) + { + case MediaType.ImageManifest: + case MediaType.ImageIndex: + if (Repository.ReferrersState == Referrers.ReferrersState.Supported) + { + // referrers API is available, no client-side indexing needed + await Repository.DeleteAsync(target, true, cancellationToken).ConfigureAwait(false); + return; + } + + await using (var manifest = await FetchAsync(target, cancellationToken).ConfigureAwait(false)) + { + await IndexReferrersForDelete(target, manifest, cancellationToken).ConfigureAwait(false); + } + break; + } + await Repository.DeleteAsync(target, true, cancellationToken).ConfigureAwait(false); + } + + /// + /// IndexReferrersForDelete indexes referrers for manifests with a subject field on manifest delete. + /// References: + /// - Latest spec: https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#deleting-manifests + /// + /// + /// + /// + private async Task IndexReferrersForDelete(Descriptor target, Stream manifestContent, CancellationToken cancellationToken = default) + { + Descriptor subject; + switch (target.MediaType) + { + case MediaType.ImageManifest: + var imageManifest = JsonSerializer.Deserialize(manifestContent); + if (imageManifest?.Subject == null) + { + // no subject, no indexing needed + return; + } + subject = imageManifest.Subject; + break; + case MediaType.ImageIndex: + var imageIndex = JsonSerializer.Deserialize(manifestContent); + if (imageIndex?.Subject == null) + { + // no subject, no indexing needed + return; + } + subject = imageIndex.Subject; + break; + default: + return; + } + + var isReferrersSupported = await Repository.PingReferrers(cancellationToken).ConfigureAwait(false); + if (isReferrersSupported) + { + // referrers API is available, no client-side indexing needed + return; + } + await UpdateReferrersIndex(subject, new Referrers.ReferrerChange(target, Referrers.ReferrerOperation.Delete), cancellationToken) + .ConfigureAwait(false); + } } diff --git a/src/OrasProject.Oras/Registry/Remote/Referrers.cs b/src/OrasProject.Oras/Registry/Remote/Referrers.cs index 29a34a3..951650e 100644 --- a/src/OrasProject.Oras/Registry/Remote/Referrers.cs +++ b/src/OrasProject.Oras/Registry/Remote/Referrers.cs @@ -12,9 +12,7 @@ // limitations under the License. using System.Collections.Generic; -using System.Linq; using OrasProject.Oras.Content; -using OrasProject.Oras.Exceptions; using OrasProject.Oras.Oci; namespace OrasProject.Oras.Registry.Remote; @@ -29,6 +27,8 @@ internal enum ReferrersState } internal record ReferrerChange(Descriptor Referrer, ReferrerOperation ReferrerOperation); + + internal const string ZeroDigest = "sha256:0000000000000000000000000000000000000000000000000000000000000000"; internal enum ReferrerOperation { diff --git a/src/OrasProject.Oras/Registry/Remote/Repository.cs b/src/OrasProject.Oras/Registry/Remote/Repository.cs index 186cdb6..ba0dfc3 100644 --- a/src/OrasProject.Oras/Registry/Remote/Repository.cs +++ b/src/OrasProject.Oras/Registry/Remote/Repository.cs @@ -76,6 +76,10 @@ internal Referrers.ReferrersState ReferrersState private RepositoryOptions _opts; + private static readonly Object _referrersPingLock = new(); + + private readonly SemaphoreSlim _referrersPingSemaphore = new SemaphoreSlim(1, 1); + /// /// Creates a client to the remote repository identified by a reference /// Example: localhost:5000/hello-world @@ -369,6 +373,72 @@ internal Reference ParseReferenceFromContentReference(string reference) public async Task MountAsync(Descriptor descriptor, string fromRepository, Func>? getContent = null, CancellationToken cancellationToken = default) => await ((IMounter)Blobs).MountAsync(descriptor, fromRepository, getContent, cancellationToken).ConfigureAwait(false); + /// + /// PingReferrers returns true if the Referrers API is available for the repository, + /// otherwise returns false + /// + /// + /// + /// + /// + internal async Task PingReferrers(CancellationToken cancellationToken = default) + { + switch (ReferrersState) + { + case Referrers.ReferrersState.Supported: + return true; + case Referrers.ReferrersState.NotSupported: + return false; + } + + await _referrersPingSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + // referrers state is unknown + // lock to limit the rate of pinging referrers API + if (ReferrersState == Referrers.ReferrersState.Supported) + { + return true; + } + + if (ReferrersState == Referrers.ReferrersState.NotSupported) + { + return false; + } + + var reference = new Reference(Options.Reference); + reference.ContentReference = Referrers.ZeroDigest; + var url = new UriFactory(reference, Options.PlainHttp).BuildReferrersUrl(); + var request = new HttpRequestMessage(HttpMethod.Get, url); + var response = await Options.HttpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + + switch (response.StatusCode) + { + case HttpStatusCode.OK: + var supported = response.Content.Headers.ContentType?.MediaType == MediaType.ImageIndex; + SetReferrersState(supported); + return supported; + case HttpStatusCode.NotFound: + var err = (ResponseException) await response.ParseErrorResponseAsync(cancellationToken) + .ConfigureAwait(false); + if (err.Errors?.First().Code == nameof(ResponseException.ErrorCode.NAME_UNKNOWN)) + { + // referrer state is unknown because the repository is not found + throw err; + } + + SetReferrersState(false); + return false; + default: + throw await response.ParseErrorResponseAsync(cancellationToken).ConfigureAwait(false); + } + } + finally + { + _referrersPingSemaphore.Release(); + } + } + /// /// SetReferrersState indicates the Referrers API state of the remote repository. true: supported; false: not supported. /// SetReferrersState is valid only when it is called for the first time. diff --git a/src/OrasProject.Oras/Registry/Remote/ResponseException.cs b/src/OrasProject.Oras/Registry/Remote/ResponseException.cs index 05bf4e0..18aed7f 100644 --- a/src/OrasProject.Oras/Registry/Remote/ResponseException.cs +++ b/src/OrasProject.Oras/Registry/Remote/ResponseException.cs @@ -15,15 +15,17 @@ using System.Collections.Generic; using System.Net; using System.Net.Http; -using System.Text; using System.Text.Json; -using System.Text.Json.Nodes; using System.Text.Json.Serialization; namespace OrasProject.Oras.Registry.Remote; public class ResponseException : HttpRequestException -{ +{ + public enum ErrorCode + { + NAME_UNKNOWN + } public class Error { [JsonPropertyName("code")] @@ -34,45 +36,45 @@ public class Error [JsonPropertyName("detail")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public JsonElement? Detail { get; set; } + public JsonElement? Detail { get; set; } } - public class ErrorResponse - { + public class ErrorResponse + { [JsonPropertyName("errors")] - public required IList Errors { get; set; } + public required IList Errors { get; set; } } public HttpMethod? Method { get; } public Uri? RequestUri { get; } - public IList? Errors { get; } - - public ResponseException(HttpResponseMessage response, string? responseBody = null) - : this(response, responseBody, null) - { - } - - public ResponseException(HttpResponseMessage response, string? responseBody, string? message) - : this(response, responseBody, response.StatusCode == HttpStatusCode.Unauthorized ? HttpRequestError.UserAuthenticationError : HttpRequestError.Unknown, message, null) - { - } - - public ResponseException(HttpResponseMessage response, string? responseBody, HttpRequestError httpRequestError, string? message, Exception? inner) - : base(httpRequestError, message, inner, response.StatusCode) - { - var request = response.RequestMessage; - Method = request?.Method; - RequestUri = request?.RequestUri; - if (responseBody != null) - { - try - { - var errorResponse = JsonSerializer.Deserialize(responseBody); - Errors = errorResponse?.Errors; - } - catch { } + public IList? Errors { get; } + + public ResponseException(HttpResponseMessage response, string? responseBody = null) + : this(response, responseBody, null) + { + } + + public ResponseException(HttpResponseMessage response, string? responseBody, string? message) + : this(response, responseBody, response.StatusCode == HttpStatusCode.Unauthorized ? HttpRequestError.UserAuthenticationError : HttpRequestError.Unknown, message, null) + { + } + + public ResponseException(HttpResponseMessage response, string? responseBody, HttpRequestError httpRequestError, string? message, Exception? inner) + : base(httpRequestError, message, inner, response.StatusCode) + { + var request = response.RequestMessage; + Method = request?.Method; + RequestUri = request?.RequestUri; + if (responseBody != null) + { + try + { + var errorResponse = JsonSerializer.Deserialize(responseBody); + Errors = errorResponse?.Errors; + } + catch { } } } } diff --git a/src/OrasProject.Oras/Registry/Remote/UriFactory.cs b/src/OrasProject.Oras/Registry/Remote/UriFactory.cs index a4b37a4..88c678e 100644 --- a/src/OrasProject.Oras/Registry/Remote/UriFactory.cs +++ b/src/OrasProject.Oras/Registry/Remote/UriFactory.cs @@ -13,6 +13,7 @@ using OrasProject.Oras.Exceptions; using System; +using System.Web; namespace OrasProject.Oras.Registry.Remote; @@ -102,6 +103,26 @@ public Uri BuildRepositoryBlobUpload() return builder.Uri; } + /// + /// Builds the URL for accessing the Referrers API + /// Format: :///v2//referrers/?artifactType= + /// + /// + /// + public Uri BuildReferrersUrl(string? artifactType = null) + { + var builder = NewRepositoryBaseBuilder(); + builder.Path += $"/referrers/{_reference.ContentReference}"; + if (!string.IsNullOrEmpty(artifactType)) + { + var query = HttpUtility.ParseQueryString(builder.Query); + query.Add("artifactType", artifactType); + builder.Query = query.ToString(); + } + + return builder.Uri; + } + /// /// Generates a UriBuilder with the base endpoint of the remote repository. /// Format: :///v2/ diff --git a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs index 0b1ac8c..05dc6e9 100644 --- a/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/ManifestStoreTest.cs @@ -22,6 +22,8 @@ using static OrasProject.Oras.Content.Digest; using Index = OrasProject.Oras.Oci.Index; using Xunit; +using Xunit.Abstractions; + namespace OrasProject.Oras.Tests.Remote; @@ -343,12 +345,38 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupported() { return new HttpResponseMessage(HttpStatusCode.BadRequest); } + response.Content = new ByteArrayContent(oldIndexBytes); response.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); if (oldIndexDeleted) response.Headers.Add(_dockerContentDigestHeader, new string[] { firstExpectedIndexReferrersDesc.Digest }); else response.Headers.Add(_dockerContentDigestHeader, new string[] { oldIndexDesc.Digest }); response.StatusCode = HttpStatusCode.OK; return response; + } + else if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{oldIndexDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + response.Content = new ByteArrayContent(oldIndexBytes); + response.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); + response.Headers.Add(_dockerContentDigestHeader, new string[] { oldIndexDesc.Digest }); + response.StatusCode = HttpStatusCode.OK; + return response; + } + else if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{firstExpectedIndexReferrersDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + response.Content = new ByteArrayContent(oldIndexBytes); + response.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); + response.Headers.Add(_dockerContentDigestHeader, new string[] { firstExpectedIndexReferrersDesc.Digest }); + response.StatusCode = HttpStatusCode.OK; + return response; + } else if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{oldIndexDesc.Digest}") { response.Headers.Add(_dockerContentDigestHeader, new[] { oldIndexDesc.Digest }); @@ -459,7 +487,7 @@ public async Task ManifestStore_PushAsyncWithSubjectAndReferrerNotSupportedWitho var cancellationToken = new CancellationToken(); var store = new ManifestStore(repo); - + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); await store.PushAsync(expectedIndexManifestDesc, new MemoryStream(expectedIndexManifestBytes), cancellationToken); Assert.Equal(Referrers.ReferrersState.NotSupported, repo.ReferrersState); @@ -552,4 +580,303 @@ public async Task ManifestStore_PushAsyncWithSubjectAndNoUpdateRequired() Assert.Equal(Referrers.ReferrersState.NotSupported, repo.ReferrersState); Assert.Equal(expectedManifestBytes, receivedManifestContent); } + + [Fact] + public async Task ManifestStore_DeleteWithSubjectWhenReferrersAPISupported() + { + var (_, manifestBytes) = RandomManifestWithSubject(); + var manifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(manifestBytes), + Size = manifestBytes.Length + }; + var manifestDeleted = false; + var httpHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Delete && req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + manifestDeleted = true; + res.StatusCode = HttpStatusCode.Accepted; + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(httpHandler), + PlainHttp = true, + }); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); + repo.SetReferrersState(true); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + await store.DeleteAsync(manifestDesc, cancellationToken); + Assert.Equal(Referrers.ReferrersState.Supported, repo.ReferrersState); + Assert.True(manifestDeleted); + } + + [Fact] + public async Task ManifestStore_DeleteWithoutSubjectWhenReferrersAPIUnknown() + { + var (_, manifestBytes) = RandomManifest(); + var manifestDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(manifestBytes), + Size = manifestBytes.Length + }; + var manifestDeleted = false; + var httpHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method != HttpMethod.Delete && req.Method != HttpMethod.Get) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + manifestDeleted = true; + res.StatusCode = HttpStatusCode.Accepted; + return res; + } + + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Content = new ByteArrayContent(manifestBytes); + res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); + res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); + return res; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(httpHandler), + PlainHttp = true, + }); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + await store.DeleteAsync(manifestDesc, cancellationToken); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); + Assert.True(manifestDeleted); + } + + [Fact] + public async Task ManifestStore_DeleteWithSubjectWhenReferrersAPINotSupported() + { + // first delete image manifest + var (manifestToDelete, manifestToDeleteBytes) = RandomManifestWithSubject(); + var manifestToDeleteDesc = new Descriptor + { + MediaType = MediaType.ImageManifest, + Digest = ComputeSHA256(manifestToDeleteBytes), + Size = manifestToDeleteBytes.Length + }; + + // then delete image index + var indexToDelete = RandomIndex(); + indexToDelete.Subject = manifestToDelete.Subject; + var indexToDeleteBytes = JsonSerializer.SerializeToUtf8Bytes(indexToDelete); + var indexToDeleteDesc = new Descriptor + { + MediaType = MediaType.ImageIndex, + Digest = ComputeSHA256(indexToDeleteBytes), + Size = indexToDeleteBytes.Length + }; + + // original referrers list + var oldReferrersList = RandomIndex(); + oldReferrersList.Manifests.Add(manifestToDeleteDesc); + oldReferrersList.Manifests.Add(indexToDeleteDesc); + var oldReferrersBytes = JsonSerializer.SerializeToUtf8Bytes(oldReferrersList); + var oldReferrersDesc = new Descriptor() + { + Digest = ComputeSHA256(oldReferrersBytes), + MediaType = MediaType.ImageIndex, + Size = oldReferrersBytes.Length + }; + + // referrers list after deleting the image manifest + var firstUpdatedReferrersList = new List(oldReferrersList.Manifests); + firstUpdatedReferrersList.Remove(manifestToDeleteDesc); + var (firstUpdatedIndexReferrersDesc, firstUpdatedIndexReferrersBytes) = Index.GenerateIndex(firstUpdatedReferrersList); + + // referrers list after deleting the index manifest + var secondUpdatedReferrersList = new List(firstUpdatedReferrersList); + secondUpdatedReferrersList.Remove(indexToDeleteDesc); + var (secondUpdatedIndexReferrersDesc, secondUpdatedIndexReferrersBytes) = Index.GenerateIndex(secondUpdatedReferrersList); + + + var manifestDeleted = false; + var oldIndexDeleted = false; + var firstUpdatedIndexDeleted = false; + var imageIndexDeleted = false; + var referrersTag = Referrers.BuildReferrersTag(manifestToDelete.Subject); + byte[]? receivedIndexContent = null; + var httpHandler = async (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var response = new HttpResponseMessage(); + response.RequestMessage = req; + if (req.Method != HttpMethod.Delete && req.Method != HttpMethod.Get && req.Method != HttpMethod.Put) + { + return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); + } + + if (req.Method == HttpMethod.Put && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{referrersTag}") + { + if (req.Content?.Headers?.ContentLength != null) + { + var buffer = new byte[req.Content.Headers.ContentLength.Value]; + (await req.Content.ReadAsByteArrayAsync(cancellationToken)).CopyTo(buffer, 0); + receivedIndexContent = buffer; + } + + if (oldIndexDeleted) + { + response.Headers.Add(_dockerContentDigestHeader, new[] { secondUpdatedIndexReferrersDesc.Digest }); + } + else + { + response.Headers.Add(_dockerContentDigestHeader, new[] { firstUpdatedIndexReferrersDesc.Digest }); + } + response.StatusCode = HttpStatusCode.Created; + return response; + } + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{referrersTag}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + if (oldIndexDeleted) + { + response.Content = new ByteArrayContent(firstUpdatedIndexReferrersBytes); + response.Headers.Add(_dockerContentDigestHeader, new string[] { firstUpdatedIndexReferrersDesc.Digest }); + } + else + { + response.Content = new ByteArrayContent(oldReferrersBytes); + response.Headers.Add(_dockerContentDigestHeader, new string[] { oldReferrersDesc.Digest }); + } + + response.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); + response.StatusCode = HttpStatusCode.OK; + return response; + } + + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestToDeleteDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageManifest)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + response.Content = new ByteArrayContent(manifestToDeleteBytes); + response.Headers.Add(_dockerContentDigestHeader, new string[] { manifestToDeleteDesc.Digest }); + response.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); + return response; + } + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{indexToDeleteDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + response.Content = new ByteArrayContent(indexToDeleteBytes); + response.Headers.Add(_dockerContentDigestHeader, new string[] { indexToDeleteDesc.Digest }); + response.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); + return response; + } + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{oldReferrersDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + response.Content = new ByteArrayContent(oldReferrersBytes); + response.Headers.Add(_dockerContentDigestHeader, new string[] { oldReferrersDesc.Digest }); + response.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); + return response; + } + + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{firstUpdatedIndexReferrersDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + response.Content = new ByteArrayContent(firstUpdatedIndexReferrersBytes); + response.Headers.Add(_dockerContentDigestHeader, new string[] { firstUpdatedIndexReferrersDesc.Digest }); + response.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); + return response; + } + if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{oldReferrersDesc.Digest}") + { + response.Headers.Add(_dockerContentDigestHeader, new[] { oldReferrersDesc.Digest }); + response.StatusCode = HttpStatusCode.Accepted; + oldIndexDeleted = true; + return response; + } + if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{firstUpdatedIndexReferrersDesc.Digest}") + { + response.Headers.Add(_dockerContentDigestHeader, new[] { firstUpdatedIndexReferrersDesc.Digest }); + response.StatusCode = HttpStatusCode.Accepted; + firstUpdatedIndexDeleted = true; + return response; + } + + if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{manifestToDeleteDesc.Digest}") + { + manifestDeleted = true; + response.StatusCode = HttpStatusCode.Accepted; + return response; + } + if (req.Method == HttpMethod.Delete && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{indexToDeleteDesc.Digest}") + { + imageIndexDeleted = true; + response.StatusCode = HttpStatusCode.Accepted; + return response; + } + return new HttpResponseMessage(HttpStatusCode.NotFound); + }; + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(httpHandler), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + var store = new ManifestStore(repo); + + // first delete the image manifest + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); + await store.DeleteAsync(manifestToDeleteDesc, cancellationToken); + Assert.Equal(Referrers.ReferrersState.NotSupported, repo.ReferrersState); + Assert.True(manifestDeleted); + Assert.True(oldIndexDeleted); + Assert.Equal(firstUpdatedIndexReferrersBytes, receivedIndexContent); + + // then delete the image index + Assert.Equal(Referrers.ReferrersState.NotSupported, repo.ReferrersState); + await store.DeleteAsync(indexToDeleteDesc, cancellationToken); + Assert.Equal(Referrers.ReferrersState.NotSupported, repo.ReferrersState); + Assert.True(imageIndexDeleted); + Assert.True(firstUpdatedIndexDeleted); + Assert.Equal(secondUpdatedIndexReferrersBytes, receivedIndexContent); + } } diff --git a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs index b893c34..260eda3 100644 --- a/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs +++ b/tests/OrasProject.Oras.Tests/Remote/RepositoryTest.cs @@ -20,13 +20,13 @@ using System.Net; using System.Net.Http.Headers; using System.Text; -using System.Text.Json; using System.Text.RegularExpressions; using System.Web; using Xunit; using static OrasProject.Oras.Content.Digest; using static OrasProject.Oras.Tests.Remote.Util.Util; using static OrasProject.Oras.Tests.Remote.Util.RandomDataGenerator; +using JsonSerializer = System.Text.Json.JsonSerializer; namespace OrasProject.Oras.Tests.Remote; @@ -391,12 +391,12 @@ public async Task Repository_DeleteAsync() { var res = new HttpResponseMessage(); res.RequestMessage = req; - if (req.Method != HttpMethod.Delete) + if (req.Method != HttpMethod.Delete && req.Method != HttpMethod.Get) { return new HttpResponseMessage(HttpStatusCode.MethodNotAllowed); } - if (req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) + if (req.Method == HttpMethod.Delete && req.RequestUri!.AbsolutePath == "/v2/test/blobs/" + blobDesc.Digest) { blobDeleted = true; res.Headers.Add(_dockerContentDigestHeader, blobDesc.Digest); @@ -404,13 +404,25 @@ public async Task Repository_DeleteAsync() return res; } - if (req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) + if (req.Method == HttpMethod.Delete && req.RequestUri!.AbsolutePath == "/v2/test/manifests/" + indexDesc.Digest) { indexDeleted = true; // no dockerContentDigestHeader header for manifest deletion res.StatusCode = HttpStatusCode.Accepted; return res; } + + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/manifests/{indexDesc.Digest}") + { + if (req.Headers.TryGetValues("Accept", out IEnumerable? values) && !values.Contains(MediaType.ImageIndex)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + res.Content = new ByteArrayContent(index); + res.Headers.Add(_dockerContentDigestHeader, new string[] { indexDesc.Digest }); + res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageIndex }); + return res; + } return new HttpResponseMessage(HttpStatusCode.NotFound); }; @@ -1745,12 +1757,12 @@ public async Task ManifestStore_ExistAsync() [Fact] public async Task ManifestStore_DeleteAsync() { - var manifest = """{"layers":[]}"""u8.ToArray(); + var (_, manifestBytes) = RandomManifest(); var manifestDesc = new Descriptor { MediaType = MediaType.ImageManifest, - Digest = ComputeSHA256(manifest), - Size = manifest.Length + Digest = ComputeSHA256(manifestBytes), + Size = manifestBytes.Length }; var manifestDeleted = false; var func = (HttpRequestMessage req, CancellationToken cancellationToken) => @@ -1773,7 +1785,7 @@ public async Task ManifestStore_DeleteAsync() { return new HttpResponseMessage(HttpStatusCode.BadRequest); } - res.Content = new ByteArrayContent(manifest); + res.Content = new ByteArrayContent(manifestBytes); res.Headers.Add(_dockerContentDigestHeader, new string[] { manifestDesc.Digest }); res.Content.Headers.Add("Content-Type", new string[] { MediaType.ImageManifest }); return res; @@ -2580,4 +2592,205 @@ public void SetReferrersState_ShouldNotThrowException_WhenSettingSameValue() var exception = Record.Exception(() => repo.ReferrersState = Referrers.ReferrersState.Supported); Assert.Null(exception); } + + [Fact] + public async Task PingReferrers_ShouldReturnTrueWhenReferrersAPISupported() + { + var mockHttpRequestHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/referrers/{Referrers.ZeroDigest}") + { + res.Content.Headers.Add("Content-Type", MediaType.ImageIndex); + res.StatusCode = HttpStatusCode.OK; + return res; + } + return new HttpResponseMessage(HttpStatusCode.Forbidden); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); + var result = await repo.PingReferrers(cancellationToken); + Assert.True(result); + Assert.Equal(Referrers.ReferrersState.Supported, repo.ReferrersState); + } + + [Fact] + public async Task PingReferrers_WaitsForSemaphoreRelease() + { + var mockHttpRequestHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/referrers/{Referrers.ZeroDigest}") + { + res.Content.Headers.Add("Content-Type", MediaType.ImageIndex); + res.StatusCode = HttpStatusCode.OK; + return res; + } + return new HttpResponseMessage(HttpStatusCode.Forbidden); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); + var ping1 = repo.PingReferrers(cancellationToken); + await Task.Delay(50, cancellationToken); + var ping2 = repo.PingReferrers(cancellationToken); + Assert.True(ping1.IsCompletedSuccessfully); + Assert.True(ping2.IsCompletedSuccessfully); + Assert.Equal(Referrers.ReferrersState.Supported, repo.ReferrersState); + } + + [Fact] + public async Task PingReferrers_LimitsConcurrency() + { + var mockHttpRequestHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/referrers/{Referrers.ZeroDigest}") + { + res.Content.Headers.Add("Content-Type", MediaType.ImageIndex); + res.StatusCode = HttpStatusCode.OK; + return res; + } + return new HttpResponseMessage(HttpStatusCode.Forbidden); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); + + var tasks = new List>(); + for (int i = 0; i < 5; ++i) + { + tasks.Add(repo.PingReferrers(cancellationToken)); + } + + var results = await Task.WhenAll(tasks); + Assert.All(results, result => Assert.True(result)); + Assert.Equal(Referrers.ReferrersState.Supported, repo.ReferrersState); + } + + [Fact] + public async Task PingReferrers_ShouldReturnFalseWhenReferrersAPINotSupportedNoContentTypeHeader() + { + var mockHttpRequestHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/referrers/{Referrers.ZeroDigest}") + { + res.StatusCode = HttpStatusCode.OK; + return res; + } + return new HttpResponseMessage(HttpStatusCode.Forbidden); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); + var result = await repo.PingReferrers(cancellationToken); + Assert.False(result); + Assert.Equal(Referrers.ReferrersState.NotSupported, repo.ReferrersState); + } + + [Fact] + public async Task PingReferrers_ShouldFailWhenReturnNotFound() + { + var mockHttpRequestHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + res.StatusCode = HttpStatusCode.NotFound; + + if (req.Method == HttpMethod.Get && req.RequestUri?.AbsolutePath == $"/v2/test/referrers/{Referrers.ZeroDigest}") + { + return res; + } + + var errors = new + { + errors = new[] + { + new + { + message = "The repository could not be found.", + code = nameof(ResponseException.ErrorCode.NAME_UNKNOWN) + } + } + }; + res.Content = new StringContent(JsonSerializer.Serialize(errors), Encoding.UTF8, "application/json"); + return res; + }; + var cancellationToken = new CancellationToken(); + + // repo abc is not found + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/abc"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); + await Assert.ThrowsAsync(async () => await repo.PingReferrers(cancellationToken)); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); + + // referrer API is not supported + var repo1 = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + Assert.Equal(Referrers.ReferrersState.Unknown, repo1.ReferrersState); + var result = await repo1.PingReferrers(cancellationToken); + Assert.False(result); + Assert.Equal(Referrers.ReferrersState.NotSupported, repo1.ReferrersState); + } + + [Fact] + public async Task PingReferrers_ShouldFailWhenBadRequestReturns() + { + var mockHttpRequestHandler = (HttpRequestMessage req, CancellationToken cancellationToken) => + { + var res = new HttpResponseMessage(); + res.RequestMessage = req; + return new HttpResponseMessage(HttpStatusCode.BadRequest); + }; + + var repo = new Repository(new RepositoryOptions() + { + Reference = Reference.Parse("localhost:5000/test"), + HttpClient = CustomClient(mockHttpRequestHandler), + PlainHttp = true, + }); + var cancellationToken = new CancellationToken(); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); + await Assert.ThrowsAsync(async () => await repo.PingReferrers(cancellationToken)); + Assert.Equal(Referrers.ReferrersState.Unknown, repo.ReferrersState); + } } diff --git a/tests/OrasProject.Oras.Tests/Remote/UriFactoryTest.cs b/tests/OrasProject.Oras.Tests/Remote/UriFactoryTest.cs new file mode 100644 index 0000000..1fc7dd3 --- /dev/null +++ b/tests/OrasProject.Oras.Tests/Remote/UriFactoryTest.cs @@ -0,0 +1,50 @@ +// Copyright The ORAS Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using OrasProject.Oras.Registry; +using OrasProject.Oras.Registry.Remote; +using Xunit; +using static OrasProject.Oras.Tests.Remote.Util.RandomDataGenerator; + +namespace OrasProject.Oras.Tests.Remote; + +public class UriFactoryTest +{ + [Fact] + public void BuildReferrersUrl_WithArtifactType_ShouldAddArtifactTypeToQueryString() + { + var desc = RandomDescriptor(); + + var reference = Reference.Parse("localhost:5000/test"); + reference.ContentReference = desc.Digest; + + const string artifactType = "doc/example"; + var expectedPath = $"referrers/{reference.ContentReference}"; + const string expectedQuery = "artifactType=doc%2fexample"; + var result = new UriFactory(reference).BuildReferrersUrl(artifactType); + Assert.Equal($"https://localhost:5000/v2/test/{expectedPath}?{expectedQuery}", result.ToString()); + } + + [Fact] + public void BuildReferrersUrl_WithoutArtifactType() + { + var desc = RandomDescriptor(); + var reference = Reference.Parse("localhost:5000/test"); + reference.ContentReference = desc.Digest; + + + var expectedPath = $"referrers/{reference.ContentReference}"; + var result = new UriFactory(reference).BuildReferrersUrl(); + Assert.Equal($"https://localhost:5000/v2/test/{expectedPath}", result.ToString()); + } +}