Skip to content

Commit 1cf5e89

Browse files
tamasvajktamasvajk
authored
Merge pull request github#16747 from tamasvajk/buildless/binary-log-extractor-2
C#: Add binlog support to buildless with source generator support
2 parents fd3089e + 4db586f commit 1cf5e89

File tree

33 files changed

+489
-97
lines changed

33 files changed

+489
-97
lines changed

‎csharp/autobuilder/Semmle.Autobuild.CSharp/BUILD.bazel

+1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ codeql_csharp_binary(
1313
"//csharp/autobuilder/Semmle.Autobuild.Shared",
1414
"//csharp/extractor/Semmle.Extraction.CSharp",
1515
"//csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching",
16+
"//csharp/extractor/Semmle.Extraction.CSharp.Driver:bin/Semmle.Extraction.CSharp.Driver",
1617
"//csharp/extractor/Semmle.Extraction.CSharp.Standalone:bin/Semmle.Extraction.CSharp.Standalone",
1718
"//csharp/extractor/Semmle.Util",
1819
"@paket.main//microsoft.build",

‎csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs

+17-1
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,10 @@ public class CSharpAutobuildOptions : AutobuildOptionsShared
1313
{
1414
private const string buildModeEnvironmentVariable = "CODEQL_EXTRACTOR_CSHARP_BUILD_MODE";
1515
internal const string ExtractorOptionBuildless = "CODEQL_EXTRACTOR_CSHARP_OPTION_BUILDLESS";
16+
internal const string ExtractorOptionBinlog = "CODEQL_EXTRACTOR_CSHARP_OPTION_BINLOG";
1617

1718
public bool Buildless { get; }
19+
public string? Binlog { get; }
1820

1921
public override Language Language => Language.CSharp;
2022

@@ -29,7 +31,7 @@ public CSharpAutobuildOptions(IBuildActions actions) : base(actions)
2931
actions.GetEnvironmentVariable(ExtractorOptionBuildless).AsBool("buildless", false) ||
3032
actions.GetEnvironmentVariable(buildModeEnvironmentVariable)?.ToLower() == "none";
3133

32-
34+
Binlog = actions.GetEnvironmentVariable(ExtractorOptionBinlog);
3335
}
3436
}
3537

@@ -114,6 +116,20 @@ private BuildScript AddBuildlessStartedDiagnostic()
114116
markdownMessage: "C# was extracted with build-mode set to 'none'. This means that all C# source in the working directory will be scanned, with build tools, such as Nuget and Dotnet CLIs, only contributing information about external dependencies.",
115117
severity: DiagnosticMessage.TspSeverity.Note
116118
));
119+
120+
// For the time being we are adding an additional message regarding the binlog usage. In the future, we might want to remove the buildless messages altogether when the binlog option is specified.
121+
if (actions.GetEnvironmentVariable(CSharpAutobuildOptions.ExtractorOptionBinlog) is not null)
122+
{
123+
AddDiagnostic(new DiagnosticMessage(
124+
Options.Language,
125+
"buildless/binlog",
126+
"C# was extracted with the experimental 'binlog' option",
127+
visibility: new DiagnosticMessage.TspVisibility(statusPage: true, cliSummaryTable: true, telemetry: true),
128+
markdownMessage: "C# was extracted with the experimental 'binlog' option.",
129+
severity: DiagnosticMessage.TspSeverity.Note
130+
));
131+
}
132+
117133
return 0;
118134
});
119135
}

‎csharp/autobuilder/Semmle.Autobuild.CSharp/Semmle.Autobuild.CSharp.csproj

+1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
<ProjectReference Include="..\..\extractor\Semmle.Util\Semmle.Util.csproj" />
77
<ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj" />
88
<ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.Standalone\Semmle.Extraction.CSharp.Standalone.csproj" />
9+
<ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.Driver\Semmle.Extraction.CSharp.Driver.csproj" />
910
<ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.DependencyFetching\Semmle.Extraction.CSharp.DependencyFetching.csproj" />
1011
<ProjectReference Include="..\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
1112
</ItemGroup>

‎csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs

+3-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,9 @@ internal class StandaloneBuildRule : IBuildRule<CSharpAutobuildOptions>
1010
{
1111
public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
1212
{
13-
return BuildScript.Create(_ => Semmle.Extraction.CSharp.Standalone.Program.Main([]));
13+
return builder.Options.Binlog is string binlog
14+
? BuildScript.Create(_ => Semmle.Extraction.CSharp.Driver.Main(["--binlog", binlog]))
15+
: BuildScript.Create(_ => Semmle.Extraction.CSharp.Standalone.Program.Main([]));
1416
}
1517
}
1618
}

‎csharp/codeql-extractor.yml

+6
Original file line numberDiff line numberDiff line change
@@ -65,3 +65,9 @@ options:
6565
- progress+++
6666
type: string
6767
pattern: "^(off|errors|warnings|(info|progress)|(debug|progress\\+)|(trace|progress\\+\\+)|progress\\+\\+\\+)$"
68+
binlog:
69+
title: Binlog
70+
description: >
71+
[EXPERIMENTAL] The value is a path to the MsBuild binary log file that should be extracted.
72+
This option only works when `--build-mode none` is also specified.
73+
type: string

‎csharp/extractor/Semmle.Extraction.CSharp.Driver/BUILD.bazel

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ codeql_csharp_binary(
88
srcs = glob([
99
"*.cs",
1010
]),
11-
visibility = ["//csharp:__pkg__"],
11+
visibility = ["//csharp:__subpackages__"],
1212
deps = [
1313
"//csharp/extractor/Semmle.Extraction.CSharp",
1414
],

‎csharp/extractor/Semmle.Extraction.CSharp/BUILD.bazel

+1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ codeql_csharp_library(
1919
"//csharp/extractor/Semmle.Extraction",
2020
"//csharp/extractor/Semmle.Extraction.CSharp.Util",
2121
"//csharp/extractor/Semmle.Util",
22+
"@paket.main//basic.compilerlog.util",
2223
"@paket.main//microsoft.build",
2324
"@paket.main//microsoft.codeanalysis.csharp",
2425
],

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/File.cs

+8-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,14 @@ public override void Populate(TextWriter trapFile)
3434
lineCounts.Total++;
3535

3636
trapFile.numlines(this, lineCounts);
37-
Context.TrapWriter.Archive(originalPath, TransformedPath, text.Encoding ?? System.Text.Encoding.Default);
37+
if (BinaryLogExtractionContext.GetAdjustedPath(Context.ExtractionContext, originalPath) is not null)
38+
{
39+
Context.TrapWriter.ArchiveContent(rawText, TransformedPath);
40+
}
41+
else
42+
{
43+
Context.TrapWriter.Archive(originalPath, TransformedPath, text.Encoding ?? System.Text.Encoding.Default);
44+
}
3845
}
3946
}
4047
else if (IsPossiblyTextFile())

‎csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs

+2-1
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,8 @@ private void DoExtractTree(SyntaxTree tree)
185185
{
186186
var stopwatch = new Stopwatch();
187187
stopwatch.Start();
188-
var sourcePath = tree.FilePath;
188+
var sourcePath = BinaryLogExtractionContext.GetAdjustedPath(ExtractionContext, tree.FilePath) ?? tree.FilePath;
189+
189190
var transformedSourcePath = PathTransformer.Transform(sourcePath);
190191

191192
var trapPath = transformedSourcePath.GetTrapPath(Logger, options.TrapCompression);

‎csharp/extractor/Semmle.Extraction.CSharp/Extractor/BinaryLogAnalyser.cs

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
using System.Collections.Generic;
2+
using Microsoft.CodeAnalysis.CSharp;
3+
using Semmle.Util;
4+
using Semmle.Util.Logging;
5+
6+
namespace Semmle.Extraction.CSharp
7+
{
8+
public class BinaryLogAnalyser : Analyser
9+
{
10+
public BinaryLogAnalyser(IProgressMonitor pm, ILogger logger, PathTransformer pathTransformer, IPathCache pathCache, bool addAssemblyTrapPrefix)
11+
: base(pm, logger, pathTransformer, pathCache, addAssemblyTrapPrefix)
12+
{
13+
}
14+
15+
public void Initialize(
16+
string cwd, string[] args, string outputPath, CSharpCompilation compilation,
17+
IEnumerable<Microsoft.CodeAnalysis.SyntaxTree> generatedSyntaxTrees,
18+
string compilationIdentifier, CommonOptions options)
19+
{
20+
base.compilation = compilation;
21+
ExtractionContext = new BinaryLogExtractionContext(
22+
cwd, args, outputPath, generatedSyntaxTrees, compilationIdentifier,
23+
Logger, PathTransformer, options.QlTest);
24+
this.options = options;
25+
LogExtractorInfo();
26+
SetReferencePaths();
27+
}
28+
}
29+
}

‎csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs

+133-33
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
using System.Text;
99
using System.Threading;
1010
using System.Threading.Tasks;
11+
using Basic.CompilerLog.Util;
1112
using Microsoft.CodeAnalysis;
1213
using Microsoft.CodeAnalysis.CSharp;
1314
using Microsoft.CodeAnalysis.Text;
@@ -102,55 +103,154 @@ public static ExitCode Run(string[] args)
102103

103104
try
104105
{
105-
if (options.ProjectsToLoad.Any())
106+
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
107+
var pathTransformer = new PathTransformer(canonicalPathCache);
108+
109+
if (options.BinaryLogPath is string binlogPath)
106110
{
107-
AddSourceFilesFromProjects(options.ProjectsToLoad, options.CompilerArguments, logger);
111+
logger.LogInfo(" Running binary log analysis.");
112+
return RunBinaryLogAnalysis(analyzerStopwatch, options, binlogPath, logger, canonicalPathCache, pathTransformer);
108113
}
109-
110-
var compilerVersion = new CompilerVersion(options);
111-
if (compilerVersion.SkipExtraction)
114+
else
112115
{
113-
logger.LogWarning($" Unrecognized compiler '{compilerVersion.SpecifiedCompiler}' because {compilerVersion.SkipReason}");
114-
return ExitCode.Ok;
116+
logger.LogInfo(" Running tracing analysis.");
117+
return RunTracingAnalysis(analyzerStopwatch, options, logger, canonicalPathCache, pathTransformer);
115118
}
119+
}
120+
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
121+
{
122+
logger.LogError($" Unhandled exception: {ex}");
123+
return ExitCode.Errors;
124+
}
125+
}
116126

117-
var workingDirectory = Directory.GetCurrentDirectory();
118-
var compilerArgs = options.CompilerArguments.ToArray();
127+
private static ExitCode RunBinaryLogAnalysis(Stopwatch stopwatch, Options options, string binlogPath, ILogger logger, CanonicalPathCache canonicalPathCache, PathTransformer pathTransformer)
128+
{
129+
logger.LogInfo($"Reading compiler calls from binary log {binlogPath}");
130+
try
131+
{
132+
using var fileStream = new FileStream(binlogPath, FileMode.Open, FileAccess.Read, FileShare.Read);
133+
using var reader = BinaryLogReader.Create(fileStream);
119134

120-
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
121-
var pathTransformer = new PathTransformer(canonicalPathCache);
135+
// Filter out compiler calls that aren't interesting for examination
136+
static bool filter(CompilerCall compilerCall)
137+
{
138+
return compilerCall.IsCSharp &&
139+
compilerCall.Kind == CompilerCallKind.Regular;
140+
}
122141

123-
using var analyser = new TracingAnalyser(new LogProgressMonitor(logger), logger, pathTransformer, canonicalPathCache, options.AssemblySensitiveTrap);
142+
var allCompilationData = reader.ReadAllCompilationData(filter);
143+
var allFailed = true;
124144

125-
var compilerArguments = CSharpCommandLineParser.Default.Parse(
126-
compilerVersion.ArgsWithResponse,
127-
workingDirectory,
128-
compilerVersion.FrameworkPath,
129-
compilerVersion.AdditionalReferenceDirectories
130-
);
145+
logger.LogInfo($" Found {allCompilationData.Count} compilations in binary log");
131146

132-
if (compilerArguments is null)
147+
foreach (var compilationData in allCompilationData)
133148
{
134-
var sb = new StringBuilder();
135-
sb.Append(" Failed to parse command line: ").AppendList(" ", compilerArgs);
136-
logger.LogError(sb.ToString());
137-
++analyser.CompilationErrors;
138-
return ExitCode.Failed;
139-
}
149+
if (compilationData.GetCompilationAfterGenerators() is not CSharpCompilation compilation)
150+
{
151+
logger.LogError(" Compilation data is not C#");
152+
continue;
153+
}
140154

141-
if (!analyser.BeginInitialize(compilerVersion.ArgsWithResponse))
142-
{
143-
logger.LogInfo("Skipping extraction since files have already been extracted");
144-
return ExitCode.Ok;
155+
var compilerCall = compilationData.CompilerCall;
156+
var diagnosticName = compilerCall.GetDiagnosticName();
157+
logger.LogInfo($" Processing compilation {diagnosticName} at {compilerCall.ProjectDirectory}");
158+
var compilerArgs = compilerCall.GetArguments();
159+
160+
var compilationIdentifierPath = string.Empty;
161+
try
162+
{
163+
compilationIdentifierPath = FileUtils.ConvertPathToSafeRelativePath(
164+
Path.GetRelativePath(Directory.GetCurrentDirectory(), compilerCall.ProjectDirectory));
165+
}
166+
catch (ArgumentException exc)
167+
{
168+
logger.LogWarning($" Failed to get relative path for {compilerCall.ProjectDirectory} from current working directory {Directory.GetCurrentDirectory()}: {exc.Message}");
169+
}
170+
171+
var args = reader.ReadCommandLineArguments(compilerCall);
172+
var generatedSyntaxTrees = compilationData.GetGeneratedSyntaxTrees();
173+
174+
using var analyser = new BinaryLogAnalyser(new LogProgressMonitor(logger), logger, pathTransformer, canonicalPathCache, options.AssemblySensitiveTrap);
175+
176+
var exit = Analyse(stopwatch, analyser, options,
177+
references => [() => compilation.References.ForEach(r => references.Add(r))],
178+
(analyser, syntaxTrees) => [() => syntaxTrees.AddRange(compilation.SyntaxTrees)],
179+
(syntaxTrees, references) => compilation,
180+
(compilation, options) => analyser.Initialize(
181+
compilerCall.ProjectDirectory,
182+
compilerArgs?.ToArray() ?? [],
183+
TracingAnalyser.GetOutputName(compilation, args),
184+
compilation,
185+
generatedSyntaxTrees,
186+
Path.Combine(compilationIdentifierPath, diagnosticName),
187+
options),
188+
() => { });
189+
190+
switch (exit)
191+
{
192+
case ExitCode.Ok:
193+
allFailed &= false;
194+
logger.LogInfo($" Compilation {diagnosticName} succeeded");
195+
break;
196+
case ExitCode.Errors:
197+
allFailed &= false;
198+
logger.LogWarning($" Compilation {diagnosticName} had errors");
199+
break;
200+
case ExitCode.Failed:
201+
logger.LogWarning($" Compilation {diagnosticName} failed");
202+
break;
203+
}
145204
}
205+
return allFailed ? ExitCode.Failed : ExitCode.Ok;
206+
}
207+
catch (IOException ex)
208+
{
209+
logger.LogError($"Failed to open binary log: {ex.Message}");
210+
return ExitCode.Failed;
211+
}
212+
}
146213

147-
return AnalyseTracing(workingDirectory, compilerArgs, analyser, compilerArguments, options, analyzerStopwatch);
214+
private static ExitCode RunTracingAnalysis(Stopwatch analyzerStopwatch, Options options, ILogger logger, CanonicalPathCache canonicalPathCache, PathTransformer pathTransformer)
215+
{
216+
if (options.ProjectsToLoad.Any())
217+
{
218+
AddSourceFilesFromProjects(options.ProjectsToLoad, options.CompilerArguments, logger);
148219
}
149-
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
220+
221+
var compilerVersion = new CompilerVersion(options);
222+
if (compilerVersion.SkipExtraction)
150223
{
151-
logger.LogError($" Unhandled exception: {ex}");
152-
return ExitCode.Errors;
224+
logger.LogWarning($" Unrecognized compiler '{compilerVersion.SpecifiedCompiler}' because {compilerVersion.SkipReason}");
225+
return ExitCode.Ok;
226+
}
227+
228+
var workingDirectory = Directory.GetCurrentDirectory();
229+
var compilerArgs = options.CompilerArguments.ToArray();
230+
using var analyser = new TracingAnalyser(new LogProgressMonitor(logger), logger, pathTransformer, canonicalPathCache, options.AssemblySensitiveTrap);
231+
var compilerArguments = CSharpCommandLineParser.Default.Parse(
232+
compilerVersion.ArgsWithResponse,
233+
workingDirectory,
234+
compilerVersion.FrameworkPath,
235+
compilerVersion.AdditionalReferenceDirectories
236+
);
237+
238+
if (compilerArguments is null)
239+
{
240+
var sb = new StringBuilder();
241+
sb.Append(" Failed to parse command line: ").AppendList(" ", compilerArgs);
242+
logger.LogError(sb.ToString());
243+
++analyser.CompilationErrors;
244+
return ExitCode.Failed;
245+
}
246+
247+
if (!analyser.BeginInitialize(compilerVersion.ArgsWithResponse))
248+
{
249+
logger.LogInfo("Skipping extraction since files have already been extracted");
250+
return ExitCode.Ok;
153251
}
252+
253+
return AnalyseTracing(workingDirectory, compilerArgs, analyser, compilerArguments, options, analyzerStopwatch);
154254
}
155255

156256
private static void AddSourceFilesFromProjects(IEnumerable<string> projectsToLoad, IList<string> compilerArguments, ILogger logger)

‎csharp/extractor/Semmle.Extraction.CSharp/Extractor/Options.cs

+8
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,11 @@ public sealed class Options : CommonOptions
3232
/// </summary>
3333
public bool AssemblySensitiveTrap { get; private set; } = false;
3434

35+
/// <summary>
36+
/// The path to the binary log file, or null if unspecified.
37+
/// </summary>
38+
public string? BinaryLogPath { get; set; }
39+
3540
public static Options CreateWithEnvironment(string[] arguments)
3641
{
3742
var options = new Options();
@@ -65,6 +70,9 @@ public override bool HandleOption(string key, string value)
6570
case "load-sources-from-project":
6671
ProjectsToLoad.Add(value);
6772
return true;
73+
case "binlog":
74+
BinaryLogPath = value;
75+
return true;
6876
default:
6977
return base.HandleOption(key, value);
7078
}

‎csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs

+2-5
Original file line numberDiff line numberDiff line change
@@ -107,11 +107,8 @@ private bool LogRoslynArgs(IEnumerable<string> roslynArgs)
107107
/// <summary>
108108
/// Determine the path of the output dll/exe.
109109
/// </summary>
110-
/// <param name="compilation">Information about the compilation.</param>
111-
/// <param name="cancel">Cancellation token required.</param>
112-
/// <returns>The filename.</returns>
113-
private static string GetOutputName(CSharpCompilation compilation,
114-
CSharpCommandLineArguments commandLineArguments)
110+
internal static string GetOutputName(CSharpCompilation compilation,
111+
CommandLineArguments commandLineArguments)
115112
{
116113
// There's no apparent way to access the output filename from the compilation,
117114
// so we need to re-parse the command line arguments.

‎csharp/extractor/Semmle.Extraction.CSharp/paket.references

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
Microsoft.Build
22
Microsoft.CodeAnalysis.CSharp
3-
3+
Basic.CompilerLog.Util

‎csharp/extractor/Semmle.Extraction/Entities/File.cs

+2-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ protected File(Context cx, string path)
88
: base(cx, path)
99
{
1010
originalPath = path;
11-
transformedPathLazy = new Lazy<PathTransformer.ITransformedPath>(() => Context.ExtractionContext.PathTransformer.Transform(originalPath));
11+
var adjustedPath = BinaryLogExtractionContext.GetAdjustedPath(Context.ExtractionContext, originalPath) ?? path;
12+
transformedPathLazy = new Lazy<PathTransformer.ITransformedPath>(() => Context.ExtractionContext.PathTransformer.Transform(adjustedPath));
1213
}
1314

1415
protected readonly string originalPath;

0 commit comments

Comments
 (0)