diff --git a/QuantConnectStubsGenerator.Tests/GeneratorTests.cs b/QuantConnectStubsGenerator.Tests/GeneratorTests.cs new file mode 100644 index 0000000..ab76549 --- /dev/null +++ b/QuantConnectStubsGenerator.Tests/GeneratorTests.cs @@ -0,0 +1,93 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using NUnit.Framework; +using QuantConnectStubsGenerator.Model; +using System.Collections.Generic; +using System.Linq; + +namespace QuantConnectStubsGenerator.Tests +{ + [TestFixture] + public class GeneratorTests + { + [TestCase("public", true)] + [TestCase("protected", true)] + [TestCase("", false)] + [TestCase("private", false)] + public void Interfaces(string interfaceModifier, bool expected) + { + var testGenerator = new TestGenerator + { + Files = new() + { + { "Test.cs", $@" +using System; + +namespace QuantConnect.Benchmarks +{{ + /// + /// Specifies how to compute a benchmark for an algorithm + /// + {interfaceModifier} interface IBenchmark + {{ + /// + /// Evaluates this benchmark at the specified time + /// + /// The time to evaluate the benchmark at + /// The value of the benchmark at the specified time + decimal Evaluate(DateTime time); + + DateTime TestProperty {{get;}} + }} +}}" } + } + }; + + var result = testGenerator.GenerateModelsPublic(); + + var namespaces = result.GetNamespaces().ToList(); + Assert.AreEqual(2, namespaces.Count); + + var baseNameSpace = namespaces.Single(x => x.Name == "QuantConnect"); + var benchmarksNameSpace = namespaces.Single(x => x.Name == "QuantConnect.Benchmarks"); + + if (!expected) + { + Assert.AreEqual(0, benchmarksNameSpace.GetClasses().Count()); + return; + } + var benchmark = benchmarksNameSpace.GetClasses().Single(); + + Assert.AreEqual("Evaluate", benchmark.Methods.Single().Name); + Assert.IsFalse(string.IsNullOrEmpty(benchmark.Methods.Single().Summary)); + + Assert.AreEqual("TestProperty", benchmark.Properties.Single().Name); + Assert.IsTrue(string.IsNullOrEmpty(benchmark.Properties.Single().Summary)); + } + + private class TestGenerator : Generator + { + public Dictionary Files { get; set; } + public TestGenerator() : base("/", "/", "/") + { + } + + protected override IEnumerable GetSyntaxTrees() + { + foreach (var fileContent in Files) + { + yield return CSharpSyntaxTree.ParseText(fileContent.Value, path: fileContent.Key); + } + } + + public ParseContext GenerateModelsPublic() + { + ParseContext context = new(); + + base.GenerateModels(context); + + return context; + } + } + } +} diff --git a/QuantConnectStubsGenerator/Generator.cs b/QuantConnectStubsGenerator/Generator.cs index 8f83092..0dbe90d 100644 --- a/QuantConnectStubsGenerator/Generator.cs +++ b/QuantConnectStubsGenerator/Generator.cs @@ -28,6 +28,90 @@ public Generator(string leanPath, string runtimePath, string outputDirectory) } public void Run() + { + // Create an empty ParseContext which will be filled with all relevant information during parsing + var context = new ParseContext(); + + GenerateModels(context); + + // Render .pyi files containing stubs for all parsed namespaces + Logger.Info($"Generating .py and .pyi files for {context.GetNamespaces().Count()} namespaces"); + foreach (var ns in context.GetNamespaces()) + { + var namespacePath = ns.Name.Replace('.', '/'); + var basePath = Path.GetFullPath($"{namespacePath}/__init__", _outputDirectory); + + RenderNamespace(ns, basePath + ".pyi"); + GeneratePyLoader(ns.Name, basePath + ".py"); + CreateTypedFileForNamespace(ns.Name); + } + + // Generate stubs for the clr module + GenerateClrStubs(); + + // Generate stubs for https://github.com/QuantConnect/Lean/blob/master/Common/AlgorithmImports.py + GenerateAlgorithmImports(); + + // Create setup.py + GenerateSetup(); + } + + protected virtual void GenerateModels(ParseContext context) + { + // Create syntax trees for all C# files + var syntaxTrees = GetSyntaxTrees().ToList(); + + // Create a compilation containing all syntax trees to retrieve semantic models from + var compilation = CSharpCompilation.Create("").AddSyntaxTrees(syntaxTrees); + + // Add all assemblies in current project to compilation to improve semantic models + foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies()) + { + if (!assembly.IsDynamic && assembly.Location != "") + { + compilation = compilation.AddReferences(MetadataReference.CreateFromFile(assembly.Location)); + } + } + + // Parse all syntax trees using all parsers + ParseSyntaxTrees(context, syntaxTrees, compilation); + ParseSyntaxTrees(context, syntaxTrees, compilation); + ParseSyntaxTrees(context, syntaxTrees, compilation); + + // Perform post-processing on all parsed classes + foreach (var ns in context.GetNamespaces()) + { + + // Remove problematic method "GetMethodInfo" from System.Reflection.RuntimeReflectionExtensions + // KW arg is `del` which python is not a fan of. TODO: Make this post filtering more generic? + if (ns.Name == "System.Reflection") + { + var reflectionClass = ns.GetClasses() + .FirstOrDefault(x => x.Type.Name == "RuntimeReflectionExtensions"); + var badMethod = reflectionClass.Methods.FirstOrDefault(x => x.Name == "GetMethodInfo"); + + reflectionClass.Methods.Remove(badMethod); + } + + + foreach (var cls in ns.GetClasses()) + { + // Remove Python implementations for methods where there is both a Python as well as a C# implementation + // The parsed C# implementation is usually more useful for autocomplete + // To improve it a little bit we move the return type of the Python implementation to the C# implementation + PostProcessClass(cls); + + // Mark methods which appear multiple times as overloaded + MarkOverloads(cls); + } + } + + // Create empty namespaces to fill gaps in between namespaces like "A.B" and "A.B.C.D" + // This is needed to make import resolution work correctly + CreateEmptyNamespaces(context); + } + + protected virtual IEnumerable GetSyntaxTrees() { // Lean projects not to generate stubs for var blacklistedProjects = new[] @@ -49,9 +133,9 @@ public void Run() // 2. Any bin CS files List blacklistedRegex = new() { - new (".*Lean\\/ADDITIONAL_STUBS\\/.*(?:DataProcessing|tests|DataQueueHandlers|Demonstration|Demostration|Algorithm)", RegexOptions.Compiled), + new (".*Lean\\/ADDITIONAL_STUBS\\/.*(?:DataProcessing|tests|DataQueueHandlers|Demonstration|Demostration|Algorithm)", RegexOptions.Compiled), new(".*\\/bin\\/", RegexOptions.Compiled), - }; + }; // Path prefixes for all blacklisted projects var blacklistedPrefixes = blacklistedProjects @@ -93,82 +177,10 @@ public void Run() Logger.Info($"Parsing {sourceFiles.Count} C# files"); - // Create syntax trees for all C# files - var syntaxTrees = sourceFiles - .Select(file => CSharpSyntaxTree.ParseText(File.ReadAllText(file), path: file)) - .ToList(); - - // Create a compilation containing all syntax trees to retrieve semantic models from - var compilation = CSharpCompilation.Create("").AddSyntaxTrees(syntaxTrees); - - // Add all assemblies in current project to compilation to improve semantic models - foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies()) - { - if (!assembly.IsDynamic && assembly.Location != "") - { - compilation = compilation.AddReferences(MetadataReference.CreateFromFile(assembly.Location)); - } - } - - // Create an empty ParseContext which will be filled with all relevant information during parsing - var context = new ParseContext(); - - // Parse all syntax trees using all parsers - ParseSyntaxTrees(context, syntaxTrees, compilation); - ParseSyntaxTrees(context, syntaxTrees, compilation); - ParseSyntaxTrees(context, syntaxTrees, compilation); - - // Perform post-processing on all parsed classes - foreach (var ns in context.GetNamespaces()) - { - - // Remove problematic method "GetMethodInfo" from System.Reflection.RuntimeReflectionExtensions - // KW arg is `del` which python is not a fan of. TODO: Make this post filtering more generic? - if (ns.Name == "System.Reflection"){ - var reflectionClass = ns.GetClasses() - .FirstOrDefault(x => x.Type.Name == "RuntimeReflectionExtensions"); - var badMethod = reflectionClass.Methods.FirstOrDefault(x => x.Name == "GetMethodInfo"); - - reflectionClass.Methods.Remove(badMethod); - } - - - foreach (var cls in ns.GetClasses()) - { - // Remove Python implementations for methods where there is both a Python as well as a C# implementation - // The parsed C# implementation is usually more useful for autocomplete - // To improve it a little bit we move the return type of the Python implementation to the C# implementation - PostProcessClass(cls); - - // Mark methods which appear multiple times as overloaded - MarkOverloads(cls); - } - } - - // Create empty namespaces to fill gaps in between namespaces like "A.B" and "A.B.C.D" - // This is needed to make import resolution work correctly - CreateEmptyNamespaces(context); - - // Render .pyi files containing stubs for all parsed namespaces - Logger.Info($"Generating .py and .pyi files for {context.GetNamespaces().Count()} namespaces"); - foreach (var ns in context.GetNamespaces()) + foreach (var file in sourceFiles) { - var namespacePath = ns.Name.Replace('.', '/'); - var basePath = Path.GetFullPath($"{namespacePath}/__init__", _outputDirectory); - - RenderNamespace(ns, basePath + ".pyi"); - GeneratePyLoader(ns.Name, basePath + ".py"); - CreateTypedFileForNamespace(ns.Name); + yield return CSharpSyntaxTree.ParseText(File.ReadAllText(file), path: file); } - - // Generate stubs for the clr module - GenerateClrStubs(); - - // Generate stubs for https://github.com/QuantConnect/Lean/blob/master/Common/AlgorithmImports.py - GenerateAlgorithmImports(); - - // Create setup.py - GenerateSetup(); } /// diff --git a/QuantConnectStubsGenerator/Parser/BaseParser.cs b/QuantConnectStubsGenerator/Parser/BaseParser.cs index 13f4a2e..7c2578a 100644 --- a/QuantConnectStubsGenerator/Parser/BaseParser.cs +++ b/QuantConnectStubsGenerator/Parser/BaseParser.cs @@ -114,7 +114,15 @@ private void ExitClass() /// protected bool HasModifier(MemberDeclarationSyntax node, string modifier) { - return node.Modifiers.Any(m => m.Text == modifier); + return HasModifier(node.Modifiers, modifier); + } + + /// + /// Check if a node has a modifier like private or static. + /// + protected bool HasModifier(SyntaxTokenList modifiers, string modifier) + { + return modifiers.Any(m => m.Text == modifier); } /// @@ -122,9 +130,23 @@ protected bool HasModifier(MemberDeclarationSyntax node, string modifier) /// protected bool ShouldSkip(MemberDeclarationSyntax node) { - return HasModifier(node, "private") || HasModifier(node, "internal") - // some classes don't any access modifier set, which means private - || !HasModifier(node, "public") && !HasModifier(node, "protected"); + if (HasModifier(node, "private") || HasModifier(node, "internal")) + { + return true; + } + + if (node.Modifiers.Count() == 0 && node.Parent != null && node.Parent.IsKind(SyntaxKind.InterfaceDeclaration)) + { + // interfaces properties/methods are public by default, so they depend on the parent really + if (node.Parent is InterfaceDeclarationSyntax interfaceDeclarationSyntax) + { + var modifiers = interfaceDeclarationSyntax.Modifiers; + return !HasModifier(modifiers, "public") && !HasModifier(modifiers, "protected"); + } + return true; + } + // some classes don't any access modifier set, which means private + return !HasModifier(node, "public") && !HasModifier(node, "protected"); } ///