Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix interface method/property handling #21

Merged
merged 1 commit into from
Apr 19, 2024
Merged

Fix interface method/property handling #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 93 additions & 0 deletions QuantConnectStubsGenerator.Tests/GeneratorTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
using NUnit.Framework;
using QuantConnectStubsGenerator.Model;
using System.Collections.Generic;
using System.Linq;

namespace QuantConnectStubsGenerator.Tests
{
[TestFixture]
public class GeneratorTests
{
[TestCase("public", true)]
[TestCase("protected", true)]
[TestCase("", false)]
[TestCase("private", false)]
public void Interfaces(string interfaceModifier, bool expected)
{
var testGenerator = new TestGenerator
{
Files = new()
{
{ "Test.cs", $@"
using System;

namespace QuantConnect.Benchmarks
{{
/// <summary>
/// Specifies how to compute a benchmark for an algorithm
/// </summary>
{interfaceModifier} interface IBenchmark
{{
/// <summary>
/// Evaluates this benchmark at the specified time
/// </summary>
/// <param name=""time"">The time to evaluate the benchmark at</param>
/// <returns>The value of the benchmark at the specified time</returns>
decimal Evaluate(DateTime time);

DateTime TestProperty {{get;}}
}}
}}" }
}
};

var result = testGenerator.GenerateModelsPublic();

var namespaces = result.GetNamespaces().ToList();
Assert.AreEqual(2, namespaces.Count);

var baseNameSpace = namespaces.Single(x => x.Name == "QuantConnect");
var benchmarksNameSpace = namespaces.Single(x => x.Name == "QuantConnect.Benchmarks");

if (!expected)
{
Assert.AreEqual(0, benchmarksNameSpace.GetClasses().Count());
return;
}
var benchmark = benchmarksNameSpace.GetClasses().Single();

Assert.AreEqual("Evaluate", benchmark.Methods.Single().Name);
Assert.IsFalse(string.IsNullOrEmpty(benchmark.Methods.Single().Summary));

Assert.AreEqual("TestProperty", benchmark.Properties.Single().Name);
Assert.IsTrue(string.IsNullOrEmpty(benchmark.Properties.Single().Summary));
}

private class TestGenerator : Generator
{
public Dictionary<string, string> Files { get; set; }
public TestGenerator() : base("/", "/", "/")
{
}

protected override IEnumerable<SyntaxTree> GetSyntaxTrees()
{
foreach (var fileContent in Files)
{
yield return CSharpSyntaxTree.ParseText(fileContent.Value, path: fileContent.Key);
}
}

public ParseContext GenerateModelsPublic()
{
ParseContext context = new();

base.GenerateModels(context);

return context;
}
}
}
}
164 changes: 88 additions & 76 deletions QuantConnectStubsGenerator/Generator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,90 @@ public Generator(string leanPath, string runtimePath, string outputDirectory)
}

public void Run()
{
// Create an empty ParseContext which will be filled with all relevant information during parsing
var context = new ParseContext();

GenerateModels(context);

// Render .pyi files containing stubs for all parsed namespaces
Logger.Info($"Generating .py and .pyi files for {context.GetNamespaces().Count()} namespaces");
foreach (var ns in context.GetNamespaces())
{
var namespacePath = ns.Name.Replace('.', '/');
var basePath = Path.GetFullPath($"{namespacePath}/__init__", _outputDirectory);

RenderNamespace(ns, basePath + ".pyi");
GeneratePyLoader(ns.Name, basePath + ".py");
CreateTypedFileForNamespace(ns.Name);
}

// Generate stubs for the clr module
GenerateClrStubs();

// Generate stubs for https://github.com/QuantConnect/Lean/blob/master/Common/AlgorithmImports.py
GenerateAlgorithmImports();

// Create setup.py
GenerateSetup();
}

protected virtual void GenerateModels(ParseContext context)
{
// Create syntax trees for all C# files
var syntaxTrees = GetSyntaxTrees().ToList();

// Create a compilation containing all syntax trees to retrieve semantic models from
var compilation = CSharpCompilation.Create("").AddSyntaxTrees(syntaxTrees);

// Add all assemblies in current project to compilation to improve semantic models
foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies())
{
if (!assembly.IsDynamic && assembly.Location != "")
{
compilation = compilation.AddReferences(MetadataReference.CreateFromFile(assembly.Location));
}
}

// Parse all syntax trees using all parsers
ParseSyntaxTrees<ClassParser>(context, syntaxTrees, compilation);
ParseSyntaxTrees<PropertyParser>(context, syntaxTrees, compilation);
ParseSyntaxTrees<MethodParser>(context, syntaxTrees, compilation);

// Perform post-processing on all parsed classes
foreach (var ns in context.GetNamespaces())
{

// Remove problematic method "GetMethodInfo" from System.Reflection.RuntimeReflectionExtensions
// KW arg is `del` which python is not a fan of. TODO: Make this post filtering more generic?
if (ns.Name == "System.Reflection")
{
var reflectionClass = ns.GetClasses()
.FirstOrDefault(x => x.Type.Name == "RuntimeReflectionExtensions");
var badMethod = reflectionClass.Methods.FirstOrDefault(x => x.Name == "GetMethodInfo");

reflectionClass.Methods.Remove(badMethod);
}


foreach (var cls in ns.GetClasses())
{
// Remove Python implementations for methods where there is both a Python as well as a C# implementation
// The parsed C# implementation is usually more useful for autocomplete
// To improve it a little bit we move the return type of the Python implementation to the C# implementation
PostProcessClass(cls);

// Mark methods which appear multiple times as overloaded
MarkOverloads(cls);
}
}

// Create empty namespaces to fill gaps in between namespaces like "A.B" and "A.B.C.D"
// This is needed to make import resolution work correctly
CreateEmptyNamespaces(context);
}

protected virtual IEnumerable<SyntaxTree> GetSyntaxTrees()
{
// Lean projects not to generate stubs for
var blacklistedProjects = new[]
Expand All @@ -49,9 +133,9 @@ public void Run()
// 2. Any bin CS files
List<Regex> blacklistedRegex = new()
{
new (".*Lean\\/ADDITIONAL_STUBS\\/.*(?:DataProcessing|tests|DataQueueHandlers|Demonstration|Demostration|Algorithm)", RegexOptions.Compiled),
new (".*Lean\\/ADDITIONAL_STUBS\\/.*(?:DataProcessing|tests|DataQueueHandlers|Demonstration|Demostration|Algorithm)", RegexOptions.Compiled),
new(".*\\/bin\\/", RegexOptions.Compiled),
};
};

// Path prefixes for all blacklisted projects
var blacklistedPrefixes = blacklistedProjects
Expand Down Expand Up @@ -93,82 +177,10 @@ public void Run()

Logger.Info($"Parsing {sourceFiles.Count} C# files");

// Create syntax trees for all C# files
var syntaxTrees = sourceFiles
.Select(file => CSharpSyntaxTree.ParseText(File.ReadAllText(file), path: file))
.ToList();

// Create a compilation containing all syntax trees to retrieve semantic models from
var compilation = CSharpCompilation.Create("").AddSyntaxTrees(syntaxTrees);

// Add all assemblies in current project to compilation to improve semantic models
foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies())
{
if (!assembly.IsDynamic && assembly.Location != "")
{
compilation = compilation.AddReferences(MetadataReference.CreateFromFile(assembly.Location));
}
}

// Create an empty ParseContext which will be filled with all relevant information during parsing
var context = new ParseContext();

// Parse all syntax trees using all parsers
ParseSyntaxTrees<ClassParser>(context, syntaxTrees, compilation);
ParseSyntaxTrees<PropertyParser>(context, syntaxTrees, compilation);
ParseSyntaxTrees<MethodParser>(context, syntaxTrees, compilation);

// Perform post-processing on all parsed classes
foreach (var ns in context.GetNamespaces())
{

// Remove problematic method "GetMethodInfo" from System.Reflection.RuntimeReflectionExtensions
// KW arg is `del` which python is not a fan of. TODO: Make this post filtering more generic?
if (ns.Name == "System.Reflection"){
var reflectionClass = ns.GetClasses()
.FirstOrDefault(x => x.Type.Name == "RuntimeReflectionExtensions");
var badMethod = reflectionClass.Methods.FirstOrDefault(x => x.Name == "GetMethodInfo");

reflectionClass.Methods.Remove(badMethod);
}


foreach (var cls in ns.GetClasses())
{
// Remove Python implementations for methods where there is both a Python as well as a C# implementation
// The parsed C# implementation is usually more useful for autocomplete
// To improve it a little bit we move the return type of the Python implementation to the C# implementation
PostProcessClass(cls);

// Mark methods which appear multiple times as overloaded
MarkOverloads(cls);
}
}

// Create empty namespaces to fill gaps in between namespaces like "A.B" and "A.B.C.D"
// This is needed to make import resolution work correctly
CreateEmptyNamespaces(context);

// Render .pyi files containing stubs for all parsed namespaces
Logger.Info($"Generating .py and .pyi files for {context.GetNamespaces().Count()} namespaces");
foreach (var ns in context.GetNamespaces())
foreach (var file in sourceFiles)
{
var namespacePath = ns.Name.Replace('.', '/');
var basePath = Path.GetFullPath($"{namespacePath}/__init__", _outputDirectory);

RenderNamespace(ns, basePath + ".pyi");
GeneratePyLoader(ns.Name, basePath + ".py");
CreateTypedFileForNamespace(ns.Name);
yield return CSharpSyntaxTree.ParseText(File.ReadAllText(file), path: file);
}

// Generate stubs for the clr module
GenerateClrStubs();

// Generate stubs for https://github.com/QuantConnect/Lean/blob/master/Common/AlgorithmImports.py
GenerateAlgorithmImports();

// Create setup.py
GenerateSetup();
}

/// <summary>
Expand Down
30 changes: 26 additions & 4 deletions QuantConnectStubsGenerator/Parser/BaseParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,17 +114,39 @@ private void ExitClass()
/// </summary>
protected bool HasModifier(MemberDeclarationSyntax node, string modifier)
{
return node.Modifiers.Any(m => m.Text == modifier);
return HasModifier(node.Modifiers, modifier);
}

/// <summary>
/// Check if a node has a modifier like private or static.
/// </summary>
protected bool HasModifier(SyntaxTokenList modifiers, string modifier)
{
return modifiers.Any(m => m.Text == modifier);
}

/// <summary>
/// We skip internal or private nodes
/// </summary>
protected bool ShouldSkip(MemberDeclarationSyntax node)
{
return HasModifier(node, "private") || HasModifier(node, "internal")
// some classes don't any access modifier set, which means private
|| !HasModifier(node, "public") && !HasModifier(node, "protected");
if (HasModifier(node, "private") || HasModifier(node, "internal"))
{
return true;
}

if (node.Modifiers.Count() == 0 && node.Parent != null && node.Parent.IsKind(SyntaxKind.InterfaceDeclaration))
{
// interfaces properties/methods are public by default, so they depend on the parent really
if (node.Parent is InterfaceDeclarationSyntax interfaceDeclarationSyntax)
{
var modifiers = interfaceDeclarationSyntax.Modifiers;
return !HasModifier(modifiers, "public") && !HasModifier(modifiers, "protected");
}
return true;
}
// some classes don't any access modifier set, which means private
return !HasModifier(node, "public") && !HasModifier(node, "protected");
}

/// <summary>
Expand Down
Loading