Skip to content

Commit 9340269

Browse files
gewarrengewarren
authored
Misc PackageIndexer updates (#408)
1 parent 72d919b commit 9340269

File tree

4 files changed

+309
-12
lines changed

4 files changed

+309
-12
lines changed

‎PackageIndexer/DotnetPackageIndex.cs

Lines changed: 297 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -73,20 +73,305 @@ params string[] feedUrls
7373
private static async Task<IReadOnlyList<(PackageIdentity, bool)>> GetPackagesFromNuGetOrgAsync(NuGetFeed feed)
7474
{
7575
Console.WriteLine("Fetching owner information...");
76-
Dictionary<string, string[]> ownerInformation = await feed.GetOwnerMappingAsync();
7776

78-
string[] packageIds = ownerInformation.Keys
79-
.ToHashSet(StringComparer.OrdinalIgnoreCase)
80-
.Where(id => IsOwnedByDotNet(ownerInformation, id) &&
81-
PackageFilter.Default.IsMatch(id))
82-
.ToArray();
77+
// TODO
78+
// 9/11/24 - Commented out until I can get owner info from Kusto.
79+
//Dictionary<string, string[]> ownerInformation = await feed.GetOwnerMappingAsync();
80+
81+
//string[] packageIds = ownerInformation.Keys
82+
// .ToHashSet(StringComparer.OrdinalIgnoreCase)
83+
// .Where(id => IsOwnedByDotNet(ownerInformation, id) &&
84+
// PackageFilter.Default.IsMatch(id))
85+
// .ToArray();
86+
87+
// Temporarily hard-code the .NET package IDs.
88+
string[] packageIds = [
89+
"Microsoft.Bcl.AsyncInterfaces",
90+
"Microsoft.Bcl.Build",
91+
"Microsoft.Bcl.Cryptography",
92+
"Microsoft.Bcl.HashCode",
93+
"Microsoft.Bcl.Numerics",
94+
"Microsoft.Bcl.TimeProvider",
95+
"Microsoft.Extensions.AmbientMetadata.Application",
96+
"Microsoft.Extensions.ApiDescription.Client",
97+
"Microsoft.Extensions.ApiDescription.Server",
98+
"Microsoft.Extensions.AsyncState",
99+
"Microsoft.Extensions.AuditReports",
100+
"Microsoft.Extensions.Caching.Abstractions",
101+
"Microsoft.Extensions.Caching.Hybrid",
102+
"Microsoft.Extensions.Caching.Memory",
103+
"Microsoft.Extensions.Caching.SqlServer",
104+
"Microsoft.Extensions.Caching.StackExchangeRedis",
105+
"Microsoft.Extensions.Compliance.Abstractions",
106+
"Microsoft.Extensions.Compliance.Redaction",
107+
"Microsoft.Extensions.Compliance.Testing",
108+
"Microsoft.Extensions.Configuration",
109+
"Microsoft.Extensions.Configuration.Abstractions",
110+
"Microsoft.Extensions.Configuration.Binder",
111+
"Microsoft.Extensions.Configuration.CommandLine",
112+
"Microsoft.Extensions.Configuration.EnvironmentVariables",
113+
"Microsoft.Extensions.Configuration.FileExtensions",
114+
"Microsoft.Extensions.Configuration.Ini",
115+
"Microsoft.Extensions.Configuration.Json",
116+
"Microsoft.Extensions.Configuration.KeyPerFile",
117+
"Microsoft.Extensions.Configuration.UserSecrets",
118+
"Microsoft.Extensions.Configuration.Xml",
119+
"Microsoft.Extensions.DependencyInjection",
120+
"Microsoft.Extensions.DependencyInjection.Abstractions",
121+
"Microsoft.Extensions.DependencyInjection.AutoActivation",
122+
"Microsoft.Extensions.DependencyInjection.Specification.Tests",
123+
"Microsoft.Extensions.DependencyModel",
124+
"Microsoft.Extensions.DiagnosticAdapter",
125+
"Microsoft.Extensions.Diagnostics",
126+
"Microsoft.Extensions.Diagnostics.Abstractions",
127+
"Microsoft.Extensions.Diagnostics.ExceptionSummarization",
128+
"Microsoft.Extensions.Diagnostics.HealthChecks",
129+
"Microsoft.Extensions.Diagnostics.HealthChecks.Abstractions",
130+
"Microsoft.Extensions.Diagnostics.HealthChecks.Common",
131+
"Microsoft.Extensions.Diagnostics.HealthChecks.EntityFrameworkCore",
132+
"Microsoft.Extensions.Diagnostics.HealthChecks.ResourceUtilization",
133+
"Microsoft.Extensions.Diagnostics.Probes",
134+
"Microsoft.Extensions.Diagnostics.ResourceMonitoring",
135+
"Microsoft.Extensions.Diagnostics.Testing",
136+
"Microsoft.Extensions.FileProviders.Abstractions",
137+
"Microsoft.Extensions.FileProviders.Composite",
138+
"Microsoft.Extensions.FileProviders.Embedded",
139+
"Microsoft.Extensions.FileProviders.Physical",
140+
"Microsoft.Extensions.FileSystemGlobbing",
141+
"Microsoft.Extensions.Hosting",
142+
"Microsoft.Extensions.Hosting.Abstractions",
143+
"Microsoft.Extensions.Hosting.Systemd",
144+
"Microsoft.Extensions.Hosting.Testing",
145+
"Microsoft.Extensions.Hosting.WindowsServices",
146+
"Microsoft.Extensions.Http",
147+
"Microsoft.Extensions.Http.Diagnostics",
148+
"Microsoft.Extensions.Http.Polly",
149+
"Microsoft.Extensions.Http.Resilience",
150+
"Microsoft.Extensions.Localization",
151+
"Microsoft.Extensions.Localization.Abstractions",
152+
"Microsoft.Extensions.Logging",
153+
"Microsoft.Extensions.Logging.Abstractions",
154+
"Microsoft.Extensions.Logging.AzureAppServices",
155+
"Microsoft.Extensions.Logging.Configuration",
156+
"Microsoft.Extensions.Logging.Console",
157+
"Microsoft.Extensions.Logging.Debug",
158+
"Microsoft.Extensions.Logging.EventLog",
159+
"Microsoft.Extensions.Logging.EventSource",
160+
"Microsoft.Extensions.Logging.TraceSource",
161+
"Microsoft.Extensions.ObjectPool",
162+
"Microsoft.Extensions.ObjectPool.DependencyInjection",
163+
"Microsoft.Extensions.Options",
164+
"Microsoft.Extensions.Options.ConfigurationExtensions",
165+
"Microsoft.Extensions.Options.Contextual",
166+
"Microsoft.Extensions.Options.DataAnnotations",
167+
"Microsoft.Extensions.Primitives",
168+
"Microsoft.Extensions.Resilience",
169+
"Microsoft.Extensions.StaticAnalysis",
170+
"Microsoft.Extensions.Telemetry",
171+
"Microsoft.Extensions.Telemetry.Abstractions",
172+
"Microsoft.Extensions.TimeProvider.Testing",
173+
"Microsoft.Extensions.WebEncoders",
174+
"Microsoft.Win32.Primitives",
175+
"Microsoft.Win32.Registry",
176+
"Microsoft.Win32.Registry.AccessControl",
177+
"Microsoft.Win32.SystemEvents",
178+
"System.AppContext",
179+
"System.Buffers",
180+
"System.CodeDom",
181+
"System.Collections",
182+
"System.Collections.Concurrent",
183+
"System.Collections.Immutable",
184+
"System.Collections.NonGeneric",
185+
"System.Collections.Specialized",
186+
"System.ComponentModel",
187+
"System.ComponentModel.Annotations",
188+
"System.ComponentModel.Composition",
189+
"System.ComponentModel.Composition.Registration",
190+
"System.ComponentModel.EventBasedAsync",
191+
"System.ComponentModel.Primitives",
192+
"System.ComponentModel.TypeConverter",
193+
"System.Composition",
194+
"System.Composition.AttributedModel",
195+
"System.Composition.Convention",
196+
"System.Composition.Hosting",
197+
"System.Composition.Runtime",
198+
"System.Composition.TypedParts",
199+
"System.Configuration.ConfigurationManager",
200+
"System.Console",
201+
"System.Data.Common",
202+
"System.Data.DataSetExtensions",
203+
"System.Data.Odbc",
204+
"System.Data.OleDb",
205+
"System.Data.SqlClient",
206+
"System.Diagnostics.Contracts",
207+
"System.Diagnostics.Debug",
208+
"System.Diagnostics.DiagnosticSource",
209+
"System.Diagnostics.EventLog",
210+
"System.Diagnostics.FileVersionInfo",
211+
"System.Diagnostics.PerformanceCounter",
212+
"System.Diagnostics.Process",
213+
"System.Diagnostics.StackTrace",
214+
"System.Diagnostics.TextWriterTraceListener",
215+
"System.Diagnostics.Tools",
216+
"System.Diagnostics.TraceSource",
217+
"System.Diagnostics.Tracing",
218+
"System.DirectoryServices",
219+
"System.DirectoryServices.AccountManagement",
220+
"System.DirectoryServices.Protocols",
221+
"System.Drawing.Common",
222+
"System.Drawing.Primitives",
223+
"System.Dynamic.Runtime",
224+
"System.Formats.Asn1",
225+
"System.Formats.Cbor",
226+
"System.Formats.Nrbf",
227+
"System.Globalization",
228+
"System.Globalization.Calendars",
229+
"System.Globalization.Extensions",
230+
"System.IO",
231+
"System.IO.Compression",
232+
"System.IO.Compression.clrcompression-arm",
233+
"System.IO.Compression.clrcompression-x64",
234+
"System.IO.Compression.clrcompression-x86",
235+
"System.IO.Compression.ZipFile",
236+
"System.IO.FileSystem",
237+
"System.IO.FileSystem.AccessControl",
238+
"System.IO.FileSystem.DriveInfo",
239+
"System.IO.FileSystem.Primitives",
240+
"System.IO.FileSystem.Watcher",
241+
"System.IO.Hashing",
242+
"System.IO.IsolatedStorage",
243+
"System.IO.MemoryMappedFiles",
244+
"System.IO.Packaging",
245+
"System.IO.Pipelines",
246+
"System.IO.Pipes",
247+
"System.IO.Pipes.AccessControl",
248+
"System.IO.Ports",
249+
"System.IO.UnmanagedMemoryStream",
250+
"System.Json",
251+
"System.Linq",
252+
"System.Linq.Expressions",
253+
"System.Linq.Parallel",
254+
"System.Linq.Queryable",
255+
"System.Management",
256+
"System.Memory",
257+
"System.Memory.Data",
258+
"System.Net.Http",
259+
"System.Net.Http.Json",
260+
"System.Net.Http.Rtc",
261+
"System.Net.Http.WinHttpHandler",
262+
"System.Net.NameResolution",
263+
"System.Net.NetworkInformation",
264+
"System.Net.Ping",
265+
"System.Net.Primitives",
266+
"System.Net.Requests",
267+
"System.Net.Security",
268+
"System.Net.ServerSentEvents",
269+
"System.Net.Sockets",
270+
"System.Net.WebHeaderCollection",
271+
"System.Net.WebSockets",
272+
"System.Net.WebSockets.Client",
273+
"System.Net.WebSockets.WebSocketProtocol",
274+
"System.Numerics.Tensors",
275+
"System.Numerics.Vectors",
276+
"System.Numerics.Vectors.WindowsRuntime",
277+
"System.ObjectModel",
278+
"System.Private.DataContractSerialization",
279+
"System.Private.Networking",
280+
"System.Private.Uri",
281+
"System.Reflection",
282+
"System.Reflection.Context",
283+
"System.Reflection.DispatchProxy",
284+
"System.Reflection.Emit",
285+
"System.Reflection.Emit.ILGeneration",
286+
"System.Reflection.Emit.Lightweight",
287+
"System.Reflection.Extensions",
288+
"System.Reflection.Metadata",
289+
"System.Reflection.MetadataLoadContext",
290+
"System.Reflection.Primitives",
291+
"System.Reflection.TypeExtensions",
292+
"System.Resources.Extensions",
293+
"System.Resources.Reader",
294+
"System.Resources.ResourceManager",
295+
"System.Resources.Writer",
296+
"System.Runtime",
297+
"System.Runtime.Caching",
298+
"System.Runtime.CompilerServices.Unsafe",
299+
"System.Runtime.CompilerServices.VisualC",
300+
"System.Runtime.Experimental",
301+
"System.Runtime.Extensions",
302+
"System.Runtime.Handles",
303+
"System.Runtime.InteropServices",
304+
"System.Runtime.InteropServices.NFloat.Internal",
305+
"System.Runtime.InteropServices.RuntimeInformation",
306+
"System.Runtime.InteropServices.WindowsRuntime",
307+
"System.Runtime.Loader",
308+
"System.Runtime.Numerics",
309+
"System.Runtime.Serialization.Formatters",
310+
"System.Runtime.Serialization.Json",
311+
"System.Runtime.Serialization.Primitives",
312+
"System.Runtime.Serialization.Schema",
313+
"System.Runtime.Serialization.Xml",
314+
"System.Security.AccessControl",
315+
"System.Security.Claims",
316+
"System.Security.Cryptography.Algorithms",
317+
"System.Security.Cryptography.Cng",
318+
"System.Security.Cryptography.Cose",
319+
"System.Security.Cryptography.Csp",
320+
"System.Security.Cryptography.Encoding",
321+
"System.Security.Cryptography.OpenSsl",
322+
"System.Security.Cryptography.Pkcs",
323+
"System.Security.Cryptography.Primitives",
324+
"System.Security.Cryptography.ProtectedData",
325+
"System.Security.Cryptography.X509Certificates",
326+
"System.Security.Cryptography.Xml",
327+
"System.Security.Permissions",
328+
"System.Security.Principal",
329+
"System.Security.Principal.Windows",
330+
"System.Security.SecureString",
331+
"System.ServiceModel.Duplex",
332+
"System.ServiceModel.Federation",
333+
"System.ServiceModel.Http",
334+
"System.ServiceModel.NetFramingBase",
335+
"System.ServiceModel.NetNamedPipe",
336+
"System.ServiceModel.NetTcp",
337+
"System.ServiceModel.Primitives",
338+
"System.ServiceModel.Security",
339+
"System.ServiceModel.Syndication",
340+
"System.ServiceModel.UnixDomainSocket",
341+
"System.ServiceProcess.ServiceController",
342+
"System.Speech",
343+
"System.Text.Encoding",
344+
"System.Text.Encoding.CodePages",
345+
"System.Text.Encoding.Extensions",
346+
"System.Text.Encodings.Web",
347+
"System.Text.Json",
348+
"System.Text.RegularExpressions",
349+
"System.Threading",
350+
"System.Threading.AccessControl",
351+
"System.Threading.Channels",
352+
"System.Threading.Overlapped",
353+
"System.Threading.Tasks",
354+
"System.Threading.Tasks.Dataflow",
355+
"System.Threading.Tasks.Extensions",
356+
"System.Threading.Tasks.Parallel",
357+
"System.Threading.Thread",
358+
"System.Threading.ThreadPool",
359+
"System.Threading.Timer",
360+
"System.ValueTuple",
361+
"System.Web.Http.Common",
362+
"System.Web.Services.Description",
363+
"System.Windows.Extensions",
364+
"System.Xml.ReaderWriter",
365+
"System.Xml.XDocument",
366+
"System.Xml.XmlDocument",
367+
"System.Xml.XmlSerializer",
368+
"System.Xml.XPath",
369+
"System.Xml.XPath.XDocument",
370+
"System.Xml.XPath.XmlDocument"
371+
];
83372

84373
Console.WriteLine($"Found {packageIds.Length:N0} package IDs owned by .NET.");
85374

86-
// Are these packages already filtered somehow? Yes - see PackageFilter.cs.
87-
// https://www.nuget.org/packages/Microsoft.NETCore.App.Ref
88-
// https://www.nuget.org/packages/Microsoft.WindowsDesktop.App.Ref
89-
90375
Console.WriteLine("Getting versions...");
91376

92377
ConcurrentBag<(PackageIdentity, bool)> identities = [];
@@ -134,6 +419,8 @@ bool usePreviewVersions
134419
if (latestPrerelease.isDeprecated)
135420
latestPrerelease = default;
136421

422+
// If the latest stable version is newer than the latest
423+
// prerelease version, don't include the prerelease version.
137424
if (latestStable != default && latestPrerelease != default)
138425
{
139426
bool stableIsNewer = VersionComparer.VersionReleaseMetadata.Compare(

‎PackageIndexer/NuGet/NuGetFeed.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -241,8 +241,9 @@ public Task<Dictionary<string, string[]>> GetOwnerMappingAsync()
241241
throw new NotSupportedException("We can only retrieve owner information for nuget.org");
242242

243243
var httpClient = new HttpClient();
244+
// TODO - this URL is no longer accessible.
244245
string url = "https://nugetprodusncazuresearch.blob.core.windows.net/v3-azuresearch-017/owners/owners.v2.json";
245-
return httpClient.GetFromJsonAsync<Dictionary<string, string[]>>(url);
246+
return httpClient.GetFromJsonAsync<Dictionary<string, string[]>>(url)!;
246247
}
247248

248249
private static bool TryGetAzureDevOpsFeed(string url, out string organization, out string project, out string feed)

‎PackageIndexer/PackageIndexer.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
<PackageReference Include="CsvHelper" Version="31.0.3" />
1212
<PackageReference Include="NuGet.Packaging" Version="6.9.1" />
1313
<PackageReference Include="NuGet.Protocol" Version="6.9.1" />
14+
<PackageReference Include="Microsoft.Azure.Kusto.Data" />
1415
</ItemGroup>
1516

1617
</Project>

‎PackageIndexer/Program.cs

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ internal static class Program
8282
private static async Task<int> Main(string[] args)
8383
{
8484
#if DEBUG
85-
args = [@"c:\users\gewarren\desktop\Package Index 0814", "preview"];
85+
args = [@"c:\users\gewarren\desktop\Package Index 0911b", "preview"];
8686
#endif
8787

8888
if ((args.Length == 0) || (args.Length > 2))
@@ -265,6 +265,14 @@ FrameworkEntry tfm
265265
StringComparison.InvariantCultureIgnoreCase
266266
);
267267

268+
// Except don't include XML file for Microsoft.Extensions.Diagnostics.ResourceMonitoring
269+
// See https://github.com/dotnet/dotnet-api-docs/pull/10395#discussion_r1758128787.
270+
if (string.Equals(
271+
packageEntry.Name,
272+
"Microsoft.Extensions.Diagnostics.ResourceMonitoring",
273+
StringComparison.InvariantCultureIgnoreCase))
274+
includeXml = false;
275+
268276
// Special case for newer assemblies - include XML documentation files.
269277
if (s_packagesWithTruthDocs.Contains(packageEntry.Name))
270278
includeXml = true;

0 commit comments

Comments
 (0)