From 7ff8507d8320f29c029bfcbd0c04fa8c169c9528 Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Wed, 30 Oct 2024 15:51:45 -0400 Subject: [PATCH] Add export-clearlydefined tool --- util/export-clearlydefined/Program.cs | 203 ++++++++++++++++++ util/export-clearlydefined/README.md | 47 ++++ .../export-clearlydefined.csproj | 15 ++ 3 files changed, 265 insertions(+) create mode 100644 util/export-clearlydefined/Program.cs create mode 100644 util/export-clearlydefined/README.md create mode 100644 util/export-clearlydefined/export-clearlydefined.csproj diff --git a/util/export-clearlydefined/Program.cs b/util/export-clearlydefined/Program.cs new file mode 100644 index 000000000..0e7438a12 --- /dev/null +++ b/util/export-clearlydefined/Program.cs @@ -0,0 +1,203 @@ +using Mono.Options; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; + +const string AppName = "export-clearlydefined"; + +bool showHelp = false; +int verbosity = 0; +string? curatedDataPath = null; +string? manifestPath = null; +var groupIdPrefixes = new HashSet { + "com.android", + "com.google", + "org.chromium", + "org.jetbrains", + "org.tensorflow", + "org.ow2.asm", +}; + +var mavenGoogleGroupIdPrefixes = new HashSet { + "com.android", + "org.chromium", + "com.google", +}; + +var excludeGroupIdPrefixes = new HashSet { + "androidx", +}; + +var options = new OptionSet { + $"Usage: {AppName} [OPTIONS]+", + "", + "Update clearlydefined/curated-data to contain bound packages and versions.", + "", + "Available Options:", + { "clear-builtin-group-ids", + $"Do not process the group id prefixes:\n{string.Join (", ", groupIdPrefixes)}", + v => groupIdPrefixes.Clear () }, + { "g|group-id=", + "Process only groups starting with {GROUP_ID}", + v => groupIdPrefixes.Add (v) }, + { "m|manifest=", + "{FILE} path to cgmanifest.json to process", + v => manifestPath = v }, + { "o=", + "clearlydefined/curated-data checkout {DIRECTORY}", + v => curatedDataPath = v }, + { "v|verbose:", + "Increase verbosity of messages, or set verbosity to {LEVEL}", + (int? v) => verbosity = v.HasValue ? v.Value : verbosity + 1 }, + { "h|?|help", + "Show this help message and exit", + v => showHelp = v != null }, +}; + +try { + options.Parse (args); + if (showHelp) { + options.WriteOptionDescriptions (Console.Out); + return; + } + + if (ErrorMissingOption ("o", curatedDataPath) || + ErrorMissingOption ("manifest", manifestPath)) { + return; + } + + Directory.CreateDirectory (curatedDataPath); + + using var manifest = ReadJsonFile (manifestPath); + foreach (var registration in manifest.RootElement.GetProperty ("registrations").EnumerateArray ()) { + if (!registration.TryGetProperty ("component", out var component) || + !component.TryGetProperty ("type", out var type) || type.GetString () != "maven" || + !component.TryGetProperty ("maven", out var maven)) { + continue; + } + + var groupId = maven.GetProperty ("groupId").GetString (); + if (ShouldSkipGroupId (groupId)) { + continue; + } + + var artifactId = maven.GetProperty ("artifactId").GetString (); + var version = maven.GetProperty ("version").GetString (); + if (string.IsNullOrWhiteSpace (artifactId) || string.IsNullOrWhiteSpace (version)) { + continue; + } + var license = registration.TryGetProperty ("license", out var licenseProp) + ? licenseProp.GetString () + : null; + AddOtherLicense (groupId, artifactId, version, GetLicenseId (license)); + } +} +catch (OptionException e) { + Environment.ExitCode = 1; + Console.Error.WriteLine ($"{AppName}: {(verbosity > 0 ? e.ToString () : e.Message)}"); + Console.Error.WriteLine ($"Try `{AppName} --help` for more information."); + return; +} + +static bool ErrorMissingOption (string option, [NotNullWhen (false)] string? value) +{ + if (!string.IsNullOrWhiteSpace (value)) { + return false; + } + Environment.ExitCode = 1; + Console.Error.WriteLine ($"{AppName}: Missing required option '{option}'."); + Console.Error.WriteLine ($"Try `{AppName} --help` for more information."); + return true; +} + +static JsonDocument ReadJsonFile (string path) +{ + var options = new JsonDocumentOptions { + }; + using var file = File.OpenRead (path); + return JsonDocument.Parse (file, options); +} + +bool ShouldSkipGroupId ([NotNullWhen (false)] string? groupId) +{ + if (string.IsNullOrWhiteSpace (groupId)) { + return true; + } + foreach (var prefix in excludeGroupIdPrefixes) { + if (groupId.StartsWith (prefix, StringComparison.OrdinalIgnoreCase)) { + return true; + } + } + foreach (var prefix in groupIdPrefixes) { + if (groupId.StartsWith (prefix, StringComparison.OrdinalIgnoreCase)) { + return false; + } + } + return groupIdPrefixes.Count > 0; +} + +string GetLicenseId (string? license) +{ + return license?.ToLowerInvariant () switch { + "the apache software license, version 2.0" + => "Apache-2.0", + "chromium and built-in dependencies" + => "BSD-3-Clause", + _ => "OTHER", + }; +} + +void AddOtherLicense (string groupId, string artifactId, string version, string licenseId) +{ + var provider = GetMavenDirectoryPart (groupId); + var dir = Path.Combine (curatedDataPath, "curations", "maven", provider, groupId); + Directory.CreateDirectory (dir); + var path = Path.Combine (dir, $"{artifactId}.yaml"); + var newPath = !File.Exists (path); + + if (!newPath && YamlHasVersion (path, version)) { + return; + } + + using var o = File.AppendText (path); + if (newPath) { + o.WriteLine ($"coordinates:"); + o.WriteLine ($" name: {artifactId}"); + o.WriteLine ($" namespace: {groupId}"); + o.WriteLine ($" provider: {provider}"); + o.WriteLine ($" type: maven"); + o.WriteLine ($"revisions:"); + } + o.WriteLine ($" {version}:"); + o.WriteLine ($" licensed:"); + o.WriteLine ($" declared: {licenseId}"); +} + +string GetMavenDirectoryPart (string groupId) +{ + foreach (var prefix in mavenGoogleGroupIdPrefixes) { + if (groupId.StartsWith (prefix, StringComparison.OrdinalIgnoreCase)) { + return "mavengoogle"; + } + } + return "mavencentral"; +} + +static bool YamlHasVersion (string path, string version) +{ + var expectedVersion = $" {version}:"; + bool sawRevisions = false; + + foreach (var line in File.ReadLines (path)) { + if (line == "revisions:") { + sawRevisions = true; + continue; + } + if (!sawRevisions) { + continue; + } + if (line == expectedVersion) { + return true; + } + } + return false; +} diff --git a/util/export-clearlydefined/README.md b/util/export-clearlydefined/README.md new file mode 100644 index 000000000..8cb851dc2 --- /dev/null +++ b/util/export-clearlydefined/README.md @@ -0,0 +1,47 @@ +# export-clearlydefined + +[Component Governance][cgdocs] is a Microsoft internal DevOps tool which scans +code to find all dependencies, and issues reports if dependencies have legal or +security issues. + + * [xamarin/AndroidX Component Governance issues][androidx-cg-issues] + +There are many reported issues concerning the license of bound +Maven packages, which hilariously wrong, e.g. +`com.google.mlkit:barcode-scanning` 17.3.0 is detected as having a license of +APSL-1.0, and ` com.android.billingclient:billing` 7.1.1 is detected as having +a license of GPL-2.0. (Just, *hilariously* wrong.) + +Unfortunately, the way to fix the Component Governance alerts is to fix the +underlying data source to mention appropriate license info: +[ClearlyDefined.io][clearlydefined-io]. This in turn requires submitting a PR +to [clearlydefined/curated-data][clearlydefined-data]. + +`util/export-clearlydefined` will add or update the YAML files to contain +`licensed: declared: OTHER` for specific Maven packages declared within +[`cgmanifest.json`](../../cgmanifest.json): Maven packages with group ids +starting with: + + * `com.android` + * `com.google` + +# Usage + +To use `export-clearlydefined`: + + 1. Checkout the [clearlydefined/curated-data][clearlydefined-data] repo. + + ***Note***: This repo ***must*** be checked out on a *case-sensitive* filesystem. + (Windows need not apply! macOS will require creating a new Disk Image with the + "APFS (Case-sensitive)" file system.) + + 2. Use `dotnet run --project util/export-clearlydefined` to update (1). + `-m` should be the path to the `cgmanifest.json` to process, and + `-o` should be the path to (1): + + dotnet run --project util/export-clearlydefined -- -m cgmanifest.json -o path/to/(1) + +[cgdocs]: https://aka.ms/cgdocs +[androidx-cg-issues]: https://devdiv.visualstudio.com/DevDiv/_componentGovernance/141724?_a=alerts&typeId=22907042&alerts-view-option=active +[clearlydefined-io]: https://clearlydefined.io/ +[clearlydefined-data]: https://github.com/clearlydefined/curated-data diff --git a/util/export-clearlydefined/export-clearlydefined.csproj b/util/export-clearlydefined/export-clearlydefined.csproj new file mode 100644 index 000000000..388e71c8a --- /dev/null +++ b/util/export-clearlydefined/export-clearlydefined.csproj @@ -0,0 +1,15 @@ + + + + Exe + net8.0 + export_clearlydefined + enable + enable + + + + + + +