From 593f9d5c7844552c62205bc3ca09faf90ac9f880 Mon Sep 17 00:00:00 2001 From: Paul Hebble Date: Thu, 20 Feb 2025 21:18:17 -0600 Subject: [PATCH] Exclude release author bots from authors list --- Netkan/Sources/Github/GithubApi.cs | 23 +-- Netkan/Sources/Github/GithubRef.cs | 6 +- Netkan/Sources/Github/GithubRelease.cs | 65 ++---- Netkan/Sources/Github/GithubReleaseAsset.cs | 21 +- Netkan/Sources/Github/GithubRepo.cs | 9 +- Netkan/Transformers/GithubTransformer.cs | 47 +++-- .../Transformers/GithubTransformerTests.cs | 188 ++++++++++-------- 7 files changed, 190 insertions(+), 169 deletions(-) diff --git a/Netkan/Sources/Github/GithubApi.cs b/Netkan/Sources/Github/GithubApi.cs index f1750dd93b..5a0a040981 100644 --- a/Netkan/Sources/Github/GithubApi.cs +++ b/Netkan/Sources/Github/GithubApi.cs @@ -7,7 +7,6 @@ using log4net; using Newtonsoft.Json; -using Newtonsoft.Json.Linq; using CKAN.NetKAN.Services; @@ -72,21 +71,21 @@ public IEnumerable GetAllReleases(GithubRef reference, bool? useP break; } Log.Debug("Parsing JSON..."); - var jsonReleases = JArray.Parse(json); - if (jsonReleases.Count < 1) + var ghReleases = JsonConvert.DeserializeObject(json) + ?.Where(ghRel => ReleaseTypeMatches(usePrerelease, ghRel.PreRelease) + // Skip releases without assets + && (reference.UseSourceArchive + || (ghRel.Assets != null + && ghRel.Assets.Any(reference.FilterMatches)))) + // Insurance against GitHub returning them in the wrong order + .OrderByDescending(ghRel => ghRel.PublishedAt) + .ToArray() + ?? Array.Empty(); + if (ghReleases.Length < 1) { // That's all folks! break; } - var ghReleases = jsonReleases - .Select(rel => new GithubRelease(reference, rel)) - .Where(ghRel => ReleaseTypeMatches(usePrerelease, ghRel.PreRelease) - // Skip releases without assets - && ghRel.Assets != null - && ghRel.Assets.Count != 0) - // Insurance against GitHub returning them in the wrong order - .OrderByDescending(ghRel => ghRel.PublishedAt) - .ToList(); foreach (var ghRel in ghReleases) { diff --git a/Netkan/Sources/Github/GithubRef.cs b/Netkan/Sources/Github/GithubRef.cs index 17d71309c6..1cdab12b89 100644 --- a/Netkan/Sources/Github/GithubRef.cs +++ b/Netkan/Sources/Github/GithubRef.cs @@ -13,7 +13,7 @@ internal sealed class GithubRef : RemoteRef public string Account { get; private set; } public string Project { get; private set; } public string Repository { get; private set; } - public Regex? Filter { get; private set; } + public Regex Filter { get; private set; } public Regex? VersionFromAsset { get; private set; } public bool UseSourceArchive { get; private set; } @@ -46,5 +46,9 @@ public GithubRef(RemoteRef remoteRef, bool useSourceArchive) throw new Kraken(string.Format(@"Could not parse reference: ""{0}""", remoteRef)); } } + + public bool FilterMatches(GithubReleaseAsset asset) + => asset.Name is string name && Filter.IsMatch(name); + } } diff --git a/Netkan/Sources/Github/GithubRelease.cs b/Netkan/Sources/Github/GithubRelease.cs index 2cba1b19f3..922d95a043 100644 --- a/Netkan/Sources/Github/GithubRelease.cs +++ b/Netkan/Sources/Github/GithubRelease.cs @@ -1,54 +1,31 @@ using System; -using System.Linq; -using System.Collections.Generic; +using System.ComponentModel; -using Newtonsoft.Json.Linq; +using Newtonsoft.Json; using CKAN.Versioning; namespace CKAN.NetKAN.Sources.Github { - internal sealed class GithubRelease + public sealed class GithubRelease { - public readonly string? Author; - public readonly ModuleVersion? Tag; - public readonly List? Assets; - public readonly bool PreRelease; - public readonly DateTime? PublishedAt; - - public GithubRelease(GithubRef reference, JToken json) - { - PreRelease = (bool?)json["prerelease"] ?? false; - Tag = (string?)json["tag_name"] is string s ? new ModuleVersion(s) : null; - if (Tag == null) - { - throw new Kraken("GitHub release missing tag!"); - } - Author = (string?)json["author"]?["login"]; - PublishedAt = (DateTime?)json["published_at"]; - Assets = reference.UseSourceArchive - && (string?)json["zipball_url"] is string sourceZIP - ? new List { - new GithubReleaseAsset(Tag.ToString(), new Uri(sourceZIP), PublishedAt), - } - : (JArray?)json["assets"] is JArray assets - && reference.Filter != null - ? assets.Select(asset => (string?)asset["name"] is string name - && reference.Filter.IsMatch(name) - && (string?)asset["browser_download_url"] is string url - && (DateTime?)asset["updated_at"] is DateTime updated - ? new GithubReleaseAsset(name, new Uri(url), updated) - : null) - .OfType() - .ToList() - : new List(); - } - - public GithubRelease(string author, ModuleVersion tag, List assets) - { - Author = author; - Tag = tag; - Assets = assets; - } + [JsonProperty("author")] + public GithubUser? Author { get; set; } + + [JsonProperty("tag_name")] + public ModuleVersion? Tag { get; set; } + + [JsonProperty("assets")] + public GithubReleaseAsset[]? Assets { get; set; } + + [JsonProperty("zipball_url")] + public Uri? SourceArchive { get; set; } + + [JsonProperty("published_at")] + public DateTime? PublishedAt { get; set; } + + [JsonProperty("prerelease")] + [DefaultValue(false)] + public bool PreRelease { get; set; } = false; } } diff --git a/Netkan/Sources/Github/GithubReleaseAsset.cs b/Netkan/Sources/Github/GithubReleaseAsset.cs index b9397400bd..44d906fae9 100644 --- a/Netkan/Sources/Github/GithubReleaseAsset.cs +++ b/Netkan/Sources/Github/GithubReleaseAsset.cs @@ -1,18 +1,21 @@ using System; +using Newtonsoft.Json; + namespace CKAN.NetKAN.Sources.Github { public sealed class GithubReleaseAsset { - public string Name { get; } - public Uri Download { get; } - public DateTime? Updated { get; } + [JsonProperty("name")] + public string? Name { get; set; } + + [JsonProperty("browser_download_url")] + public Uri? Download { get; set; } + + [JsonProperty("updated_at")] + public DateTime? Updated { get; set; } - public GithubReleaseAsset(string name, Uri download, DateTime? updated) - { - Name = name; - Download = download; - Updated = updated; - } + [JsonProperty("uploader")] + public GithubUser? Uploader { get; set; } } } diff --git a/Netkan/Sources/Github/GithubRepo.cs b/Netkan/Sources/Github/GithubRepo.cs index 7dc72b2c80..85ee193344 100644 --- a/Netkan/Sources/Github/GithubRepo.cs +++ b/Netkan/Sources/Github/GithubRepo.cs @@ -53,6 +53,13 @@ public class GithubUser public string? Login { get; set; } [JsonProperty("type")] - public string? Type { get; set; } + public GithubUserType? Type { get; set; } = GithubUserType.User; + } + + public enum GithubUserType + { + User, + Organization, + Bot, } } diff --git a/Netkan/Transformers/GithubTransformer.cs b/Netkan/Transformers/GithubTransformer.cs index 995a0197b4..198bc6705a 100644 --- a/Netkan/Transformers/GithubTransformer.cs +++ b/Netkan/Transformers/GithubTransformer.cs @@ -67,14 +67,27 @@ public IEnumerable Transform(Metadata metadata, TransformOptions? opts releases = releases.Take(opts.Releases.Value); } bool returnedAny = false; - foreach (GithubRelease rel in releases) + foreach (var rel in releases) { - if (ghRef.VersionFromAsset != null && rel.Assets != null) + if (ghRef.UseSourceArchive && rel.Tag != null) + { + returnedAny = true; + yield return TransformOne(metadata, metadata.Json(), ghRef, ghRepo, rel, + new GithubReleaseAsset() + { + Name = rel.Tag.ToString(), + Download = rel.SourceArchive, + Updated = rel.PublishedAt, + Uploader = rel.Author, + }, + rel.Tag.ToString()); + } + else if (ghRef.VersionFromAsset != null && rel.Assets != null) { Log.DebugFormat("Found version_from_asset regex, inflating all assets"); foreach (var asset in rel.Assets) { - var match = ghRef.VersionFromAsset.Match(asset.Name); + var match = ghRef.VersionFromAsset.Match(asset.Name ?? ""); if (!match.Success) { continue; @@ -92,13 +105,15 @@ public IEnumerable Transform(Metadata metadata, TransformOptions? opts } else if (rel.Assets != null && rel.Tag != null) { - if (rel.Assets.Count > 1) + var assets = rel.Assets.Where(ghRef.FilterMatches).ToArray(); + if (assets.Length > 1) { Log.WarnFormat("Multiple assets found for {0} {1} without `version_from_asset`", metadata.Identifier, rel.Tag); } returnedAny = true; - yield return TransformOne(metadata, metadata.Json(), ghRef, ghRepo, rel, rel.Assets.First(), rel.Tag.ToString()); + yield return TransformOne(metadata, metadata.Json(), ghRef, ghRepo, rel, + assets.First(), rel.Tag.ToString()); } } if (!returnedAny) @@ -159,7 +174,10 @@ private Metadata TransformOne(Metadata metadata, json.SafeAdd("version", version); json.SafeAdd("author", () => GetAuthors(ghRepo, ghRelease)); json.Remove("$kref"); - json.SafeAdd("download", ghAsset.Download.ToString()); + if (ghAsset.Download is Uri url) + { + json.SafeAdd("download", url.ToString()); + } if (ghRef.UseSourceArchive) { // https://docs.github.com/en/rest/repos/contents#download-a-repository-archive-zip @@ -241,17 +259,16 @@ public static void SetRepoResources(GithubRepo repo, JObject resources) : _api.GetRepo(new GithubRef($"#/ckan/github/{r.ParentRepo.FullName}", false))) .Reverse() - .SelectMany(r => r.Owner?.Type switch + .Select(r => r.Owner) + .Append(release.Author) + .OfType() + .SelectMany(u => u.Type switch { - userType => Enumerable.Repeat(r.Owner.Login, 1), - orgType => _api.getOrgMembers(r.Owner) - .Select(u => u.Login), - _ => Enumerable.Empty() + GithubUserType.User => Enumerable.Repeat(u.Login, 1), + GithubUserType.Organization => _api.getOrgMembers(u) + .Select(u => u.Login), + GithubUserType.Bot or _ => Enumerable.Empty(), }) - .Append(release.Author) .ToJValueOrJArray(); - - private const string userType = "User"; - private const string orgType = "Organization"; } } diff --git a/Tests/NetKAN/Transformers/GithubTransformerTests.cs b/Tests/NetKAN/Transformers/GithubTransformerTests.cs index 1e631d6f3c..32f3d10286 100644 --- a/Tests/NetKAN/Transformers/GithubTransformerTests.cs +++ b/Tests/NetKAN/Transformers/GithubTransformerTests.cs @@ -1,9 +1,10 @@ using System; -using System.Collections.Generic; using System.Linq; + using Moq; using Newtonsoft.Json.Linq; using NUnit.Framework; + using CKAN.Versioning; using CKAN.NetKAN.Model; using CKAN.NetKAN.Sources.Github; @@ -29,71 +30,79 @@ public void setupApiMockup() }); mApi.Setup(i => i.GetLatestRelease(It.IsAny(), false)) - .Returns(new GithubRelease( - "ExampleProject", - new ModuleVersion("1.0"), - new List - { - new GithubReleaseAsset( - "download", - new Uri("http://github.example/download"), - null - ) - } - )); + .Returns(new GithubRelease() + { + Author = new GithubUser() { Login = "ExampleProject" }, + Tag = new ModuleVersion("1.0"), + Assets = new GithubReleaseAsset[] + { + new GithubReleaseAsset() + { + Name = "download.zip", + Download = new Uri("http://github.example/download") + }, + }, + }); mApi.Setup(i => i.GetAllReleases(It.IsAny(), false)) .Returns(new GithubRelease[] { - new GithubRelease( - "ExampleProject", - new ModuleVersion("1.0"), - new List - { - new GithubReleaseAsset( - "download", - new Uri("http://github.example/download/1.0"), - null - ) - } - ), - new GithubRelease("ExampleProject", - new ModuleVersion("1.1"), - new List - { - new GithubReleaseAsset( - "download", - new Uri("http://github.example/download/1.1"), - null - ) - } - ), - new GithubRelease("ExampleProject", - new ModuleVersion("1.2"), - new List - { - new GithubReleaseAsset( - "ExampleProject_1.2-1.8.1.zip", - new Uri("http://github.example/download/1.2/ExampleProject_1.2-1.8.1.zip"), - null - ) - } - ), - new GithubRelease("ExampleProject", - new ModuleVersion("1.3"), - new List - { - new GithubReleaseAsset( - "ExampleProject_1.2-1.8.1.zip", - new Uri("http://github.example/download/1.2/ExampleProject_1.2-1.8.1.zip"), - null - ), - new GithubReleaseAsset( - "ExampleProject_1.2-1.9.1.zip", - new Uri("http://github.example/download/1.2/ExampleProject_1.2-1.9.1.zip"), - null - ) - } - ), + new GithubRelease() + { + Author = new GithubUser() { Login = "ExampleProject" }, + Tag = new ModuleVersion("1.0"), + Assets = new GithubReleaseAsset[] + { + new GithubReleaseAsset() + { + Name = "download.zip", + Download = new Uri("http://github.example/download/1.0"), + }, + }, + }, + new GithubRelease() + { + Author = new GithubUser() { Login = "ExampleProject" }, + Tag = new ModuleVersion("1.1"), + Assets = new GithubReleaseAsset[] + { + new GithubReleaseAsset() + { + Name = "download.zip", + Download = new Uri("http://github.example/download/1.1"), + }, + }, + }, + new GithubRelease() + { + Author = new GithubUser() { Login = "ExampleProject" }, + Tag = new ModuleVersion("1.2"), + Assets = new GithubReleaseAsset[] + { + new GithubReleaseAsset() + { + Name = "ExampleProject_1.2-1.8.1.zip", + Download = new Uri("http://github.example/download/1.2/ExampleProject_1.2-1.8.1.zip"), + }, + }, + }, + new GithubRelease() + { + Author = new GithubUser() { Login = "ExampleProject" }, + Tag = new ModuleVersion("1.3"), + Assets = new GithubReleaseAsset[] + { + new GithubReleaseAsset() + { + Name = "ExampleProject_1.2-1.8.1.zip", + Download = new Uri("http://github.example/download/1.2/ExampleProject_1.2-1.8.1.zip"), + }, + new GithubReleaseAsset() + { + Name = "ExampleProject_1.2-1.9.1.zip", + Download = new Uri("http://github.example/download/1.2/ExampleProject_1.2-1.9.1.zip"), + } + }, + }, }); apiMockUp = mApi; @@ -138,32 +147,37 @@ public void Transform_DownloadURLWithEncodedCharacter_DontDoubleEncode() }); mApi.Setup(i => i.GetLatestRelease(It.IsAny(), false)) - .Returns(new GithubRelease( - "DestructionEffects", - new ModuleVersion("v1.8,0"), - new List - { - new GithubReleaseAsset( - "DestructionEffects.1.8.0_0412018.zip", - new Uri("https://github.com/jrodrigv/DestructionEffects/releases/download/v1.8%2C0/DestructionEffects.1.8.0_0412018.zip"), - null - ) - } - )); + .Returns(new GithubRelease() + { + Author = new GithubUser() { Login = "DestructionEffects" }, + Tag = new ModuleVersion("v1.8,0"), + Assets = new GithubReleaseAsset[] + { + new GithubReleaseAsset() + { + Name = "DestructionEffects.1.8.0_0412018.zip", + Download = new Uri("https://github.com/jrodrigv/DestructionEffects/releases/download/v1.8%2C0/DestructionEffects.1.8.0_0412018.zip"), + } + }, + }); mApi.Setup(i => i.GetAllReleases(It.IsAny(), false)) - .Returns(new GithubRelease[] { new GithubRelease( - "DestructionEffects", - new ModuleVersion("v1.8,0"), - new List - { - new GithubReleaseAsset( - "DestructionEffects.1.8.0_0412018.zip", - new Uri("https://github.com/jrodrigv/DestructionEffects/releases/download/v1.8%2C0/DestructionEffects.1.8.0_0412018.zip"), - null - ) - } - )}); + .Returns(new GithubRelease[] + { + new GithubRelease() + { + Author = new GithubUser() { Login = "DestructionEffects" }, + Tag = new ModuleVersion("v1.8,0"), + Assets = new GithubReleaseAsset[] + { + new GithubReleaseAsset() + { + Name = "DestructionEffects.1.8.0_0412018.zip", + Download = new Uri("https://github.com/jrodrigv/DestructionEffects/releases/download/v1.8%2C0/DestructionEffects.1.8.0_0412018.zip"), + }, + }, + }, + }); ITransformer sut = new GithubTransformer(mApi.Object, false);