Skip to content

Commit de58ee5

Browse files
tamasvajktamasvajk
authored
Merge pull request #16225 from tamasvajk/buildless/resx
C#: Add resource generator
2 parents a1a93c7 + f20812d commit de58ee5

File tree

31 files changed

+831
-292
lines changed

31 files changed

+831
-292
lines changed

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

+13-35
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,8 @@ void exitCallback(int ret, string msg, bool silent)
152152
var sourceGenerators = new ISourceGenerator[]
153153
{
154154
new ImplicitUsingsGenerator(fileContent, logger, tempWorkingDirectory),
155-
new WebViewGenerator(fileProvider, fileContent, dotnet, this, logger, tempWorkingDirectory, usedReferences.Keys)
155+
new RazorGenerator(fileProvider, fileContent, dotnet, this, logger, tempWorkingDirectory, usedReferences.Keys),
156+
new ResxGenerator(fileProvider, fileContent, dotnet, this, logger, nugetPackageRestorer, tempWorkingDirectory, usedReferences.Keys),
156157
};
157158

158159
foreach (var sourceGenerator in sourceGenerators)
@@ -255,35 +256,6 @@ private void RemoveNugetAnalyzerReferences()
255256
}
256257
}
257258

258-
private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet<AssemblyLookupLocation> dllLocations, ISet<string> frameworkLocations)
259-
{
260-
var versionFolders = GetPackageVersionSubDirectories(frameworkPath);
261-
if (versionFolders.Length > 1)
262-
{
263-
var versions = string.Join(", ", versionFolders.Select(d => d.Name));
264-
logger.LogDebug($"Found multiple {frameworkType} DLLs in NuGet packages at {frameworkPath}. Using the latest version ({versionFolders[0].Name}) from: {versions}.");
265-
}
266-
267-
var selectedFrameworkFolder = versionFolders.FirstOrDefault()?.FullName;
268-
if (selectedFrameworkFolder is null)
269-
{
270-
logger.LogDebug($"Found {frameworkType} DLLs in NuGet packages at {frameworkPath}, but no version folder was found.");
271-
selectedFrameworkFolder = frameworkPath;
272-
}
273-
274-
dllLocations.Add(selectedFrameworkFolder);
275-
frameworkLocations.Add(selectedFrameworkFolder);
276-
logger.LogDebug($"Found {frameworkType} DLLs in NuGet packages at {selectedFrameworkFolder}.");
277-
}
278-
279-
private static DirectoryInfo[] GetPackageVersionSubDirectories(string packagePath)
280-
{
281-
return new DirectoryInfo(packagePath)
282-
.EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
283-
.OrderByDescending(d => d.Name) // TODO: Improve sorting to handle pre-release versions.
284-
.ToArray();
285-
}
286-
287259
private void RemoveFrameworkNugetPackages(ISet<AssemblyLookupLocation> dllLocations, int fromIndex = 0)
288260
{
289261
var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks;
@@ -310,10 +282,12 @@ private void AddNetFrameworkDlls(ISet<AssemblyLookupLocation> dllLocations, ISet
310282
{
311283
foreach (var fp in frameworkPaths)
312284
{
313-
dotnetFrameworkVersionVariantCount += GetPackageVersionSubDirectories(fp.Path!).Length;
285+
dotnetFrameworkVersionVariantCount += NugetPackageRestorer.GetOrderedPackageVersionSubDirectories(fp.Path!).Length;
314286
}
315287

316-
SelectNewestFrameworkPath(frameworkPath.Path, ".NET Framework", dllLocations, frameworkLocations);
288+
var folder = nugetPackageRestorer.GetNewestNugetPackageVersionFolder(frameworkPath.Path, ".NET Framework");
289+
dllLocations.Add(folder);
290+
frameworkLocations.Add(folder);
317291
RemoveFrameworkNugetPackages(dllLocations, frameworkPath.Index + 1);
318292
return;
319293
}
@@ -331,7 +305,7 @@ private void AddNetFrameworkDlls(ISet<AssemblyLookupLocation> dllLocations, ISet
331305
if (runtimeLocation is null)
332306
{
333307
logger.LogInfo("No .NET Desktop Runtime location found. Attempting to restore the .NET Framework reference assemblies manually.");
334-
runtimeLocation = nugetPackageRestorer.TryRestoreLatestNetFrameworkReferenceAssemblies();
308+
runtimeLocation = nugetPackageRestorer.TryRestore(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies);
335309
}
336310
}
337311

@@ -371,7 +345,9 @@ private void AddAspNetCoreFrameworkDlls(ISet<AssemblyLookupLocation> dllLocation
371345
// First try to find ASP.NET Core assemblies in the NuGet packages
372346
if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework) is string aspNetCorePackage)
373347
{
374-
SelectNewestFrameworkPath(aspNetCorePackage, "ASP.NET Core", dllLocations, frameworkLocations);
348+
var folder = nugetPackageRestorer.GetNewestNugetPackageVersionFolder(aspNetCorePackage, "ASP.NET Core");
349+
dllLocations.Add(folder);
350+
frameworkLocations.Add(folder);
375351
return;
376352
}
377353

@@ -387,7 +363,9 @@ private void AddMicrosoftWindowsDesktopDlls(ISet<AssemblyLookupLocation> dllLoca
387363
{
388364
if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework) is string windowsDesktopApp)
389365
{
390-
SelectNewestFrameworkPath(windowsDesktopApp, "Windows Desktop App", dllLocations, frameworkLocations);
366+
var folder = nugetPackageRestorer.GetNewestNugetPackageVersionFolder(windowsDesktopApp, "Windows Desktop App");
367+
dllLocations.Add(folder);
368+
frameworkLocations.Add(folder);
391369
}
392370
}
393371

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

+5
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
22
{
33
internal class EnvironmentVariableNames
44
{
5+
/// <summary>
6+
/// Controls whether to generate source files from resources (`.resx`).
7+
/// </summary>
8+
public const string ResourceGeneration = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_EXTRACT_RESOURCES";
9+
510
/// <summary>
611
/// Controls whether to generate source files from Asp.Net Core views (`.cshtml`, `.razor`).
712
/// </summary>

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileProvider.cs

+3
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ public class FileProvider
2222
private readonly Lazy<string[]> nugetConfigs;
2323
private readonly Lazy<string[]> globalJsons;
2424
private readonly Lazy<string[]> razorViews;
25+
private readonly Lazy<string[]> resources;
2526
private readonly Lazy<string?> rootNugetConfig;
2627

2728
public FileProvider(DirectoryInfo sourceDir, ILogger logger)
@@ -44,6 +45,7 @@ public FileProvider(DirectoryInfo sourceDir, ILogger logger)
4445
nugetConfigs = new Lazy<string[]>(() => allNonBinary.Value.SelectFileNamesByName("nuget.config").ToArray());
4546
globalJsons = new Lazy<string[]>(() => allNonBinary.Value.SelectFileNamesByName("global.json").ToArray());
4647
razorViews = new Lazy<string[]>(() => SelectTextFileNamesByExtension("razor view", ".cshtml", ".razor"));
48+
resources = new Lazy<string[]>(() => SelectTextFileNamesByExtension("resource", ".resx"));
4749

4850
rootNugetConfig = new Lazy<string?>(() => all.SelectRootFiles(SourceDir).SelectFileNamesByName("nuget.config").FirstOrDefault());
4951
}
@@ -116,5 +118,6 @@ private FileInfo[] GetAllFiles()
116118
public string? RootNugetConfig => rootNugetConfig.Value;
117119
public IEnumerable<string> GlobalJsons => globalJsons.Value;
118120
public ICollection<string> RazorViews => razorViews.Value;
121+
public ICollection<string> Resources => resources.Value;
119122
}
120123
}

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs

+3
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ public partial record class RestoreResult(bool Success, IList<string> Output)
3030
private readonly Lazy<bool> hasNugetPackageSourceError = new(() => Output.Any(s => s.Contains("NU1301")));
3131
public bool HasNugetPackageSourceError => hasNugetPackageSourceError.Value;
3232

33+
private readonly Lazy<bool> hasNugetNoStablePackageVersionError = new(() => Output.Any(s => s.Contains("NU1103")));
34+
public bool HasNugetNoStablePackageVersionError => hasNugetNoStablePackageVersionError.Value;
35+
3336
private static IEnumerable<string> GetFirstGroupOnMatch(Regex regex, IEnumerable<string> lines) =>
3437
lines
3538
.Select(line => regex.Match(line))

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

+91-27
Original file line numberDiff line numberDiff line change
@@ -48,16 +48,48 @@ public NugetPackageRestorer(
4848
missingPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath(fileProvider.SourceDir.FullName, "missingpackages"), "missing package", logger);
4949
}
5050

51-
public string? TryRestoreLatestNetFrameworkReferenceAssemblies()
51+
public string? TryRestore(string package)
5252
{
53-
if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies))
53+
if (TryRestorePackageManually(package))
5454
{
55-
return DependencyManager.GetPackageDirectory(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, missingPackageDirectory.DirInfo);
55+
var packageDir = DependencyManager.GetPackageDirectory(package, missingPackageDirectory.DirInfo);
56+
if (packageDir is not null)
57+
{
58+
return GetNewestNugetPackageVersionFolder(packageDir, package);
59+
}
5660
}
5761

5862
return null;
5963
}
6064

65+
public string GetNewestNugetPackageVersionFolder(string packagePath, string packageFriendlyName)
66+
{
67+
var versionFolders = GetOrderedPackageVersionSubDirectories(packagePath);
68+
if (versionFolders.Length > 1)
69+
{
70+
var versions = string.Join(", ", versionFolders.Select(d => d.Name));
71+
logger.LogDebug($"Found multiple {packageFriendlyName} DLLs in NuGet packages at {packagePath}. Using the latest version ({versionFolders[0].Name}) from: {versions}.");
72+
}
73+
74+
var selectedFrameworkFolder = versionFolders.FirstOrDefault()?.FullName;
75+
if (selectedFrameworkFolder is null)
76+
{
77+
logger.LogDebug($"Found {packageFriendlyName} DLLs in NuGet packages at {packagePath}, but no version folder was found.");
78+
selectedFrameworkFolder = packagePath;
79+
}
80+
81+
logger.LogDebug($"Found {packageFriendlyName} DLLs in NuGet packages at {selectedFrameworkFolder}.");
82+
return selectedFrameworkFolder;
83+
}
84+
85+
public static DirectoryInfo[] GetOrderedPackageVersionSubDirectories(string packagePath)
86+
{
87+
return new DirectoryInfo(packagePath)
88+
.EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
89+
.OrderByDescending(d => d.Name) // TODO: Improve sorting to handle pre-release versions.
90+
.ToArray();
91+
}
92+
6193
public HashSet<AssemblyLookupLocation> Restore()
6294
{
6395
var assemblyLookupLocations = new HashSet<AssemblyLookupLocation>();
@@ -408,7 +440,8 @@ private static IEnumerable<string> GetRestoredPackageDirectoryNames(DirectoryInf
408440
.Select(d => Path.GetFileName(d).ToLowerInvariant());
409441
}
410442

411-
private bool TryRestorePackageManually(string package, string? nugetConfig = null, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj, bool tryWithoutNugetConfig = true)
443+
private bool TryRestorePackageManually(string package, string? nugetConfig = null, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj,
444+
bool tryWithoutNugetConfig = true, bool tryPrereleaseVersion = true)
412445
{
413446
logger.LogInfo($"Restoring package {package}...");
414447
using var tempDir = new TemporaryDirectory(
@@ -430,59 +463,87 @@ private bool TryRestorePackageManually(string package, string? nugetConfig = nul
430463
return false;
431464
}
432465

433-
var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
434-
if (!res.Success)
466+
var res = TryRestorePackageManually(package, nugetConfig, tempDir, tryPrereleaseVersion);
467+
if (res.Success)
468+
{
469+
return true;
470+
}
471+
472+
if (tryWithoutNugetConfig && res.HasNugetPackageSourceError && nugetConfig is not null)
435473
{
436-
if (tryWithoutNugetConfig && res.HasNugetPackageSourceError && nugetConfig is not null)
474+
logger.LogDebug($"Trying to restore '{package}' without nuget.config.");
475+
// Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
476+
res = TryRestorePackageManually(package, nugetConfig: null, tempDir, tryPrereleaseVersion);
477+
if (res.Success)
437478
{
438-
// Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
439-
res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
479+
return true;
440480
}
481+
}
441482

442-
// TODO: the restore might fail, we could retry with
443-
// - a prerelease (*-* instead of *) version of the package,
444-
// - a different target framework moniker.
483+
logger.LogInfo($"Failed to restore nuget package {package}");
484+
return false;
485+
}
486+
487+
private RestoreResult TryRestorePackageManually(string package, string? nugetConfig, TemporaryDirectory tempDir, bool tryPrereleaseVersion)
488+
{
489+
var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig, ForceReevaluation: true));
490+
491+
if (!res.Success && tryPrereleaseVersion && res.HasNugetNoStablePackageVersionError)
492+
{
493+
logger.LogDebug($"Failed to restore nuget package {package} because no stable version was found.");
494+
TryChangePackageVersion(tempDir.DirInfo, "*-*");
445495

496+
res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig, ForceReevaluation: true));
446497
if (!res.Success)
447498
{
448-
logger.LogInfo($"Failed to restore nuget package {package}");
449-
return false;
499+
TryChangePackageVersion(tempDir.DirInfo, "*");
450500
}
451501
}
452502

453-
return true;
503+
return res;
454504
}
455505

456506
private void TryChangeTargetFrameworkMoniker(DirectoryInfo tempDir)
507+
{
508+
TryChangeProjectFile(tempDir, TargetFramework(), $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", "target framework moniker");
509+
}
510+
511+
private void TryChangePackageVersion(DirectoryInfo tempDir, string newVersion)
512+
{
513+
TryChangeProjectFile(tempDir, PackageReferenceVersion(), $"Version=\"{newVersion}\"", "package reference version");
514+
}
515+
516+
private bool TryChangeProjectFile(DirectoryInfo projectDir, Regex pattern, string replacement, string patternName)
457517
{
458518
try
459519
{
460-
logger.LogInfo($"Changing the target framework moniker in {tempDir.FullName}...");
520+
logger.LogDebug($"Changing the {patternName} in {projectDir.FullName}...");
461521

462-
var csprojs = tempDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
522+
var csprojs = projectDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
463523
if (csprojs.Length != 1)
464524
{
465-
logger.LogError($"Could not find the .csproj file in {tempDir.FullName}, count = {csprojs.Length}");
466-
return;
525+
logger.LogError($"Could not find the .csproj file in {projectDir.FullName}, count = {csprojs.Length}");
526+
return false;
467527
}
468528

469529
var csproj = csprojs[0];
470530
var content = File.ReadAllText(csproj.FullName);
471-
var matches = TargetFramework().Matches(content);
531+
var matches = pattern.Matches(content);
472532
if (matches.Count == 0)
473533
{
474-
logger.LogError($"Could not find target framework in {csproj.FullName}");
475-
}
476-
else
477-
{
478-
content = TargetFramework().Replace(content, $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", 1);
479-
File.WriteAllText(csproj.FullName, content);
534+
logger.LogError($"Could not find the {patternName} in {csproj.FullName}");
535+
return false;
480536
}
537+
538+
content = pattern.Replace(content, replacement, 1);
539+
File.WriteAllText(csproj.FullName, content);
540+
return true;
481541
}
482542
catch (Exception exc)
483543
{
484-
logger.LogError($"Failed to update target framework in {tempDir.FullName}: {exc}");
544+
logger.LogError($"Failed to change the {patternName} in {projectDir.FullName}: {exc}");
485545
}
546+
return false;
486547
}
487548

488549
private static async Task ExecuteGetRequest(string address, HttpClient httpClient, CancellationToken cancellationToken)
@@ -664,6 +725,9 @@ private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
664725
[GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
665726
private static partial Regex TargetFramework();
666727

728+
[GeneratedRegex(@"Version=""(\*|\*-\*)""", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
729+
private static partial Regex PackageReferenceVersion();
730+
667731
[GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
668732
private static partial Regex LegacyNugetPackage();
669733

0 commit comments

Comments
 (0)