diff --git a/test/EFCore.Jet.FunctionalTests/EFCore.Jet.FunctionalTests.csproj b/test/EFCore.Jet.FunctionalTests/EFCore.Jet.FunctionalTests.csproj index 9f0453bf..3a4cd267 100644 --- a/test/EFCore.Jet.FunctionalTests/EFCore.Jet.FunctionalTests.csproj +++ b/test/EFCore.Jet.FunctionalTests/EFCore.Jet.FunctionalTests.csproj @@ -8,9 +8,11 @@ - + + %(RecursiveDir)%(Filename)%(Extension) + - + diff --git a/test/EFCore.Jet.FunctionalTests/Properties/AssemblyInfo.cs b/test/EFCore.Jet.FunctionalTests/Properties/AssemblyInfo.cs index a39e37a3..684e3ba9 100644 --- a/test/EFCore.Jet.FunctionalTests/Properties/AssemblyInfo.cs +++ b/test/EFCore.Jet.FunctionalTests/Properties/AssemblyInfo.cs @@ -14,3 +14,5 @@ [assembly: TestCaseOrderer("EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit." + nameof(AscendingTestCaseOrderer), "EntityFrameworkCore.Jet.FunctionalTests")] #endif + +[assembly: TestFramework("EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit." + nameof(JetXunitTestFramework), "EntityFrameworkCore.Jet.FunctionalTests")] \ No newline at end of file diff --git a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/AscendingTestCaseOrderer.cs b/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/AscendingTestCaseOrderer.cs deleted file mode 100644 index f37e2c78..00000000 --- a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/AscendingTestCaseOrderer.cs +++ /dev/null @@ -1,16 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using Xunit.Abstractions; -using Xunit.Sdk; - -// ReSharper disable once CheckNamespace -namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit -{ - public class AscendingTestCaseOrderer : ITestCaseOrderer - { - public IEnumerable OrderTestCases(IEnumerable testCases) - where TTestCase : ITestCase - => testCases.OrderBy(c => c.DisplayName, StringComparer.OrdinalIgnoreCase); - } -} \ No newline at end of file diff --git a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/AscendingTestCollectionOrderer.cs b/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/AscendingTestCollectionOrderer.cs deleted file mode 100644 index 94f7baad..00000000 --- a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/AscendingTestCollectionOrderer.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using Xunit; -using Xunit.Abstractions; - -// ReSharper disable once CheckNamespace -namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit -{ - public class AscendingTestCollectionOrderer : ITestCollectionOrderer - { - public IEnumerable OrderTestCollections(IEnumerable testCollections) - => testCollections.OrderBy(c => c.DisplayName, StringComparer.OrdinalIgnoreCase); - } -} \ No newline at end of file diff --git a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/ExceptionTestCaseOrderer.cs b/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/ExceptionTestCaseOrderer.cs deleted file mode 100644 index 25dfe54a..00000000 --- a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/ExceptionTestCaseOrderer.cs +++ /dev/null @@ -1,41 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Text; -using Microsoft.EntityFrameworkCore.Internal; -using Xunit.Abstractions; -using Xunit.Sdk; - -// ReSharper disable once CheckNamespace -namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit -{ - public class ExceptionTestCaseOrderer : ITestCaseOrderer - { - private readonly string[] _testCaseOrder = - { - "Select_GetValueOrDefault_on_DateTime", - "Select_byte_constant", - "Select_DTO_with_member_init_distinct_in_subquery_translated_to_server", - "Select_Except_reference_projection", - "Select_Union", - "Select_DTO_with_member_init_distinct_in_subquery_translated_to_server_2", - "Select_bool_closure", - }; - - public IEnumerable OrderTestCases(IEnumerable testCases) - where TTestCase : ITestCase - { - var orderedTestCases = testCases.OrderBy(c => Array.IndexOf(_testCaseOrder, c.TestMethod.Method.Name)).ToList(); - - var builder = new StringBuilder() - .AppendLine("Test Case Order:") - .AppendLine(string.Join(Environment.NewLine, orderedTestCases.Select(c => c.TestMethod.Method.Name))); - - Debug.WriteLine(builder); - Console.WriteLine(builder); - - return orderedTestCases; - } - } -} \ No newline at end of file diff --git a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/RandomTestCaseOrderer.cs b/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/RandomTestCaseOrderer.cs deleted file mode 100644 index e06d849f..00000000 --- a/test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/RandomTestCaseOrderer.cs +++ /dev/null @@ -1,30 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Text; -using Xunit.Abstractions; -using Xunit.Sdk; - -// ReSharper disable once CheckNamespace -namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit -{ - public class RandomTestCaseOrderer : ITestCaseOrderer - { - public IEnumerable OrderTestCases(IEnumerable testCases) - where TTestCase : ITestCase - { - var random = new Random(); - var orderedTestCases = testCases.OrderBy(c => random.NextDouble()).ToList(); - - var builder = new StringBuilder() - .AppendLine("Test Case Order:") - .AppendLine(string.Join(Environment.NewLine, orderedTestCases.Select(c => c.TestMethod.Method.Name))); - - Debug.WriteLine(builder); - Console.WriteLine(builder); - - return orderedTestCases; - } - } -} \ No newline at end of file diff --git a/test/EFCore.Jet.Tests/EFCore.Jet.Tests.csproj b/test/EFCore.Jet.Tests/EFCore.Jet.Tests.csproj index 89a2c4aa..89bbb6b0 100644 --- a/test/EFCore.Jet.Tests/EFCore.Jet.Tests.csproj +++ b/test/EFCore.Jet.Tests/EFCore.Jet.Tests.csproj @@ -7,10 +7,6 @@ AnyCPU;x86 - - - - PreserveNewest diff --git a/test/Shared/TestUtilities/Attributes/AccessProviderTypeVariation.cs b/test/Shared/TestUtilities/Attributes/AccessProviderTypeVariation.cs new file mode 100644 index 00000000..b4b7a76f --- /dev/null +++ b/test/Shared/TestUtilities/Attributes/AccessProviderTypeVariation.cs @@ -0,0 +1,14 @@ +using System; + +namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities; + +[Flags] +public enum AccessProviderTypeVariation +{ + None = 0, + X86 = 1 << 0, + X64 = 1 << 1, + Odbc = 1 << 2, + OleDb = 1 << 3, + All = -1, +} \ No newline at end of file diff --git a/test/Shared/TestUtilities/Attributes/TestRunnerCrashAttribute.cs b/test/Shared/TestUtilities/Attributes/TestRunnerCrashAttribute.cs new file mode 100644 index 00000000..b9d61e2a --- /dev/null +++ b/test/Shared/TestUtilities/Attributes/TestRunnerCrashAttribute.cs @@ -0,0 +1,43 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore.TestUtilities.Xunit; + +namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities; + +/// +/// Marks a test method or class that is known to crash the test runner. +/// +[AttributeUsage(AttributeTargets.Class | AttributeTargets.Method, AllowMultiple = true)] +public class TestRunnerCrashAttribute : Attribute, ITestCondition +{ + public const string DefaultSkipReason = "The test is known to crash the test runner."; + + protected AccessProviderTypeVariation[] AccessProviderTypeVariations { get; } + + public TestRunnerCrashAttribute(params AccessProviderTypeVariation[] accessProviderTypeVariations) + { + AccessProviderTypeVariations = accessProviderTypeVariations.Length > 0 + ? accessProviderTypeVariations + : new[] { AccessProviderTypeVariation.All }; + } + + public virtual ValueTask IsMetAsync() + { + // Implement and enable if we want to filter tests by specific runtime scenarios. + var currentVariation = AccessProviderTypeVariation.All; // AppConfig.AccessProviderTypeVariation; + var isMet = AccessProviderTypeVariations.Any(v => v.HasFlag(currentVariation)); + + if (!isMet && string.IsNullOrEmpty(Skip)) + { + Skip = DefaultSkipReason; + } + + return new ValueTask(isMet); + } + + public virtual string SkipReason + => Skip; + + public virtual string Skip { get; set; } +} \ No newline at end of file diff --git a/test/Shared/TestUtilities/Xunit/AscendingTestCaseOrderer.cs b/test/Shared/TestUtilities/Xunit/AscendingTestCaseOrderer.cs new file mode 100644 index 00000000..4e4b4242 --- /dev/null +++ b/test/Shared/TestUtilities/Xunit/AscendingTestCaseOrderer.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit.Abstractions; +using Xunit.Sdk; + +// ReSharper disable once CheckNamespace +namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit; + +public class AscendingTestCaseOrderer : ITestCaseOrderer +{ + public IEnumerable OrderTestCases(IEnumerable testCases) + where TTestCase : ITestCase + => testCases.OrderBy(c => c.DisplayName, StringComparer.OrdinalIgnoreCase); +} \ No newline at end of file diff --git a/test/Shared/TestUtilities/Xunit/AscendingTestCollectionOrderer.cs b/test/Shared/TestUtilities/Xunit/AscendingTestCollectionOrderer.cs new file mode 100644 index 00000000..fbc2d8d1 --- /dev/null +++ b/test/Shared/TestUtilities/Xunit/AscendingTestCollectionOrderer.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit; +using Xunit.Abstractions; + +// ReSharper disable once CheckNamespace +namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit; + +public class AscendingTestCollectionOrderer : ITestCollectionOrderer +{ + public IEnumerable OrderTestCollections(IEnumerable testCollections) + => testCollections.OrderBy(c => c.DisplayName, StringComparer.OrdinalIgnoreCase); +} \ No newline at end of file diff --git a/test/Shared/TestUtilities/Xunit/ExceptionTestCaseOrderer.cs b/test/Shared/TestUtilities/Xunit/ExceptionTestCaseOrderer.cs new file mode 100644 index 00000000..6c0699bd --- /dev/null +++ b/test/Shared/TestUtilities/Xunit/ExceptionTestCaseOrderer.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using Microsoft.EntityFrameworkCore.Internal; +using Xunit.Abstractions; +using Xunit.Sdk; + +// ReSharper disable once CheckNamespace +namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit; + +public class ExceptionTestCaseOrderer : ITestCaseOrderer +{ + private readonly string[] _testCaseOrder = + { + "Select_GetValueOrDefault_on_DateTime", + "Select_byte_constant", + "Select_DTO_with_member_init_distinct_in_subquery_translated_to_server", + "Select_Except_reference_projection", + "Select_Union", + "Select_DTO_with_member_init_distinct_in_subquery_translated_to_server_2", + "Select_bool_closure", + }; + + public IEnumerable OrderTestCases(IEnumerable testCases) + where TTestCase : ITestCase + { + var orderedTestCases = testCases.OrderBy(c => Array.IndexOf(_testCaseOrder, c.TestMethod.Method.Name)).ToList(); + + var builder = new StringBuilder() + .AppendLine("Test Case Order:") + .AppendLine(string.Join(Environment.NewLine, orderedTestCases.Select(c => c.TestMethod.Method.Name))); + + Debug.WriteLine(builder); + Console.WriteLine(builder); + + return orderedTestCases; + } +} \ No newline at end of file diff --git a/test/Shared/TestUtilities/Xunit/JetConditionalFactDiscoverer.cs b/test/Shared/TestUtilities/Xunit/JetConditionalFactDiscoverer.cs new file mode 100644 index 00000000..2640704d --- /dev/null +++ b/test/Shared/TestUtilities/Xunit/JetConditionalFactDiscoverer.cs @@ -0,0 +1,23 @@ +using Microsoft.EntityFrameworkCore.TestUtilities.Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit; + +public class JetConditionalFactDiscoverer : ConditionalFactDiscoverer +{ + public JetConditionalFactDiscoverer(IMessageSink messageSink) + : base(messageSink) + { + } + + protected override IXunitTestCase CreateTestCase( + ITestFrameworkDiscoveryOptions discoveryOptions, + ITestMethod testMethod, + IAttributeInfo factAttribute) + => new JetConditionalFactTestCase( + DiagnosticMessageSink, + discoveryOptions.MethodDisplayOrDefault(), + discoveryOptions.MethodDisplayOptionsOrDefault(), + testMethod); +} \ No newline at end of file diff --git a/test/Shared/TestUtilities/Xunit/JetConditionalFactTestCase.cs b/test/Shared/TestUtilities/Xunit/JetConditionalFactTestCase.cs new file mode 100644 index 00000000..e0373803 --- /dev/null +++ b/test/Shared/TestUtilities/Xunit/JetConditionalFactTestCase.cs @@ -0,0 +1,47 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore.TestUtilities.Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit; + +/// +/// We cannot inherit from ConditionalFactTestCase, because it's sealed. +/// +public sealed class JetConditionalFactTestCase : XunitTestCase +{ + [Obsolete("Called by the de-serializer; should only be called by deriving classes for de-serialization purposes")] + public JetConditionalFactTestCase() + { + } + + public JetConditionalFactTestCase( + IMessageSink diagnosticMessageSink, + TestMethodDisplay defaultMethodDisplay, + TestMethodDisplayOptions defaultMethodDisplayOptions, + ITestMethod testMethod, + object[] testMethodArguments = null) + : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod, testMethodArguments) + { + } + + public override async Task RunAsync( + IMessageSink diagnosticMessageSink, + IMessageBus messageBus, + object[] constructorArguments, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) + => await XunitTestCaseExtensions.TrySkipAsync(this, messageBus) + ? new RunSummary { Total = 1, Skipped = 1 } + : await new JetXunitTestCaseRunner( + this, + DisplayName, + SkipReason, + constructorArguments, + TestMethodArguments, + messageBus, + aggregator, + cancellationTokenSource).RunAsync(); +} \ No newline at end of file diff --git a/test/Shared/TestUtilities/Xunit/JetConditionalTheoryDiscoverer.cs b/test/Shared/TestUtilities/Xunit/JetConditionalTheoryDiscoverer.cs new file mode 100644 index 00000000..327f2882 --- /dev/null +++ b/test/Shared/TestUtilities/Xunit/JetConditionalTheoryDiscoverer.cs @@ -0,0 +1,40 @@ +using System.Collections.Generic; +using Microsoft.EntityFrameworkCore.TestUtilities.Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit; + +public class JetConditionalTheoryDiscoverer : ConditionalTheoryDiscoverer +{ + public JetConditionalTheoryDiscoverer(IMessageSink messageSink) + : base(messageSink) + { + } + + protected override IEnumerable CreateTestCasesForTheory( + ITestFrameworkDiscoveryOptions discoveryOptions, + ITestMethod testMethod, + IAttributeInfo theoryAttribute) + { + yield return new JetConditionalTheoryTestCase( + DiagnosticMessageSink, + discoveryOptions.MethodDisplayOrDefault(), + discoveryOptions.MethodDisplayOptionsOrDefault(), + testMethod); + } + + protected override IEnumerable CreateTestCasesForDataRow( + ITestFrameworkDiscoveryOptions discoveryOptions, + ITestMethod testMethod, + IAttributeInfo theoryAttribute, + object[] dataRow) + { + yield return new JetConditionalFactTestCase( + DiagnosticMessageSink, + discoveryOptions.MethodDisplayOrDefault(), + discoveryOptions.MethodDisplayOptionsOrDefault(), + testMethod, + dataRow); + } +} \ No newline at end of file diff --git a/test/Shared/TestUtilities/Xunit/JetConditionalTheoryTestCase.cs b/test/Shared/TestUtilities/Xunit/JetConditionalTheoryTestCase.cs new file mode 100644 index 00000000..6cdcd945 --- /dev/null +++ b/test/Shared/TestUtilities/Xunit/JetConditionalTheoryTestCase.cs @@ -0,0 +1,47 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore.TestUtilities.Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit; + +/// +/// We cannot inherit from ConditionalTheoryTestCase, because it's sealed. +/// +public sealed class JetConditionalTheoryTestCase : XunitTheoryTestCase +{ + [Obsolete("Called by the de-serializer; should only be called by deriving classes for de-serialization purposes")] + public JetConditionalTheoryTestCase() + { + } + + public JetConditionalTheoryTestCase( + IMessageSink diagnosticMessageSink, + TestMethodDisplay defaultMethodDisplay, + TestMethodDisplayOptions defaultMethodDisplayOptions, + ITestMethod testMethod) + : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod) + { + } + + public override async Task RunAsync( + IMessageSink diagnosticMessageSink, + IMessageBus messageBus, + object[] constructorArguments, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) + => await XunitTestCaseExtensions.TrySkipAsync(this, messageBus) + ? new RunSummary { Total = 1, Skipped = 1 } + : await new JetXunitTheoryTestCaseRunner( + this, + DisplayName, + SkipReason, + constructorArguments, + diagnosticMessageSink, + messageBus, + aggregator, + cancellationTokenSource) + .RunAsync(); +} \ No newline at end of file diff --git a/test/Shared/TestUtilities/Xunit/JetXunitTestCaseRunner.cs b/test/Shared/TestUtilities/Xunit/JetXunitTestCaseRunner.cs new file mode 100644 index 00000000..71eb8785 --- /dev/null +++ b/test/Shared/TestUtilities/Xunit/JetXunitTestCaseRunner.cs @@ -0,0 +1,135 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit; + +public class JetXunitTestCaseRunner : XunitTestCaseRunner +{ + public const string TestRunnerCrashCacheDirectory = "TestRunnerCrashCache"; + public const string AutoSkipPrefix = "[AutoSkip]"; + public const string AutoSkipTestRunnerCrashingTestsEnvironmentVariableName = "EFCoreJet_AutoSkipTestRunnerCrashingTests"; + + public virtual bool EnableAutoSkipTestsKnownToCrashTestRunner + => (Environment.GetEnvironmentVariable(AutoSkipTestRunnerCrashingTestsEnvironmentVariableName)?.ToLowerInvariant() ?? "true") != "false"; + + public JetXunitTestCaseRunner(IXunitTestCase testCase, + string displayName, + string skipReason, + object[] constructorArguments, + object[] testMethodArguments, + IMessageBus messageBus, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) + : base( + testCase, + displayName, + skipReason, + constructorArguments, + testMethodArguments, + messageBus, + aggregator, + cancellationTokenSource) + { + } + + protected override XunitTestRunner CreateTestRunner(ITest test, + IMessageBus messageBus, + Type testClass, + object[] constructorArguments, + MethodInfo testMethod, + object[] testMethodArguments, + string skipReason, + IReadOnlyList beforeAfterAttributes, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) + => new JetXunitTestRunner( + test, + messageBus, + testClass, + constructorArguments, + testMethod, + testMethodArguments, + skipReason, + beforeAfterAttributes, + new ExceptionAggregator(aggregator), + cancellationTokenSource); + + /// + /// `TestRunner<TTestCase>.RunAsync()` is not virtual, so we need to override this method here to call our own + /// `JetXunitTestRunner.RunAsync()` implementation. + /// > + protected override async Task RunTestAsync() + { + if (EnableAutoSkipTestsKnownToCrashTestRunner) + { + AutoSkipTestsKnownToCrashTestRunner(); + } + + return await RunWithCrashDetection( + () => ((JetXunitTestRunner)CreateTestRunner( + CreateTest(TestCase, DisplayName), + MessageBus, + TestClass, + ConstructorArguments, + TestMethod, + TestMethodArguments, + SkipReason, + BeforeAfterAttributes, + Aggregator, + CancellationTokenSource)) + .RunAsync()); + } + + protected virtual async Task RunWithCrashDetection(Func> func) + { + Directory.CreateDirectory(TestRunnerCrashCacheDirectory); + + var filePath = Path.Combine(TestRunnerCrashCacheDirectory, $"{DateTime.UtcNow:yyyyMMdd'_'HHmmss.fffffff}_{(Environment.Is64BitProcess ? "x64" : "x86")}_{Guid.NewGuid()}.txt"); + var contents = $"{TestCase.TestMethod.TestClass.Class.Name}\t{TestCase.TestMethod.Method.Name}"; + await File.WriteAllTextAsync(filePath, contents); + + var result = await func(); + + File.Delete(filePath); + + return result; + } + + protected virtual void AutoSkipTestsKnownToCrashTestRunner() + { + if (IsTestKnownToCrashTestRunner(TestCase)) + { + SkipReason = $"{AutoSkipPrefix} {TestRunnerCrashAttribute.DefaultSkipReason}"; + } + } + + protected virtual bool IsTestKnownToCrashTestRunner(ITestCase testCase) + { + if (File.Exists(JetXunitTestFramework.TestsKnownToCrashTestRunnerFilePath)) + { + foreach (var line in File.ReadLines(JetXunitTestFramework.TestsKnownToCrashTestRunnerFilePath)) + { + var parts = line.Split('\t'); + if (parts.Length >= 2) + { + var testClass = parts[^2]; + var testMethod = parts[^1]; + + if (testClass == testCase.TestMethod.TestClass.Class.Name && + testMethod == testCase.TestMethod.Method.Name) + { + return true; + } + } + } + } + + return false; + } +} \ No newline at end of file diff --git a/test/Shared/TestUtilities/Xunit/JetXunitTestFramework.cs b/test/Shared/TestUtilities/Xunit/JetXunitTestFramework.cs new file mode 100644 index 00000000..bb6cf3a7 --- /dev/null +++ b/test/Shared/TestUtilities/Xunit/JetXunitTestFramework.cs @@ -0,0 +1,44 @@ +using System; +using System.IO; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit; + +public class JetXunitTestFramework : XunitTestFramework +{ + public const string TestsKnownToCrashTestRunnerFilePath = "./../../../TestsKnownToCrashTestRunner.txt"; + public const string DetectCrashesOfPreviousRunsEnvironmentVariableName = "EFCoreJet_DetectCrashesOfPreviousRuns"; + + public virtual bool EnableDetectCrashesOfPreviousRuns + => Environment.GetEnvironmentVariable(DetectCrashesOfPreviousRunsEnvironmentVariableName)?.ToLowerInvariant() == "true"; + + public JetXunitTestFramework(IMessageSink messageSink) : base(messageSink) + { + } + + protected override ITestFrameworkDiscoverer CreateDiscoverer(IAssemblyInfo assemblyInfo) + { + if (EnableDetectCrashesOfPreviousRuns) + { + DetectCrashesOfPreviousRuns(); + } + + return new JetXunitTestFrameworkDiscoverer(assemblyInfo, SourceInformationProvider, DiagnosticMessageSink); + } + + protected virtual void DetectCrashesOfPreviousRuns() + { + if (!Directory.Exists(JetXunitTestCaseRunner.TestRunnerCrashCacheDirectory)) + return; + + foreach (var filePath in Directory.EnumerateFiles(JetXunitTestCaseRunner.TestRunnerCrashCacheDirectory, "*.txt")) + { + var contents = File.ReadAllText(filePath); + contents = $"{string.Join('\t', Path.GetFileNameWithoutExtension(filePath).Split('_'))}\t{contents}\n"; + File.AppendAllText(TestsKnownToCrashTestRunnerFilePath, contents); + + File.Delete(filePath); + } + } +} \ No newline at end of file diff --git a/test/Shared/TestUtilities/Xunit/JetXunitTestFrameworkDiscoverer.cs b/test/Shared/TestUtilities/Xunit/JetXunitTestFrameworkDiscoverer.cs new file mode 100644 index 00000000..f790be7e --- /dev/null +++ b/test/Shared/TestUtilities/Xunit/JetXunitTestFrameworkDiscoverer.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.EntityFrameworkCore.TestUtilities.Xunit; +using Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit; + +public class JetXunitTestFrameworkDiscoverer : XunitTestFrameworkDiscoverer +{ + public JetXunitTestFrameworkDiscoverer( + IAssemblyInfo assemblyInfo, + ISourceInformationProvider sourceProvider, + IMessageSink diagnosticMessageSink, + IXunitTestCollectionFactory collectionFactory = null) + : base( + assemblyInfo, + sourceProvider, + diagnosticMessageSink, + collectionFactory) + { + // Prime the cache with our own discoverers, so they get used over the original ones from EF Core. + DiscovererTypeCache.Add(typeof(ConditionalFactAttribute), typeof(JetConditionalFactDiscoverer)); + DiscovererTypeCache.Add(typeof(ConditionalTheoryAttribute), typeof(JetConditionalTheoryDiscoverer)); + } + + protected override bool IsValidTestClass(ITypeInfo type) + => base.IsValidTestClass(type) && + IsTestConditionMet(type); + + protected virtual bool IsTestConditionMet(ITypeInfo type) where TType : ITestCondition + => GetTestConditions(type).Aggregate(true, (current, next) => current && next.IsMetAsync().Result); + + protected virtual IEnumerable GetTestConditions(ITypeInfo type) where TType : ITestCondition + => type.GetCustomAttributes(typeof(TType)) + .Select(attribute => (TType)Activator.CreateInstance(typeof(TType), attribute.GetConstructorArguments().ToArray())) + .Cast(); +} \ No newline at end of file diff --git a/test/Shared/TestUtilities/Xunit/JetXunitTestRunner.cs b/test/Shared/TestUtilities/Xunit/JetXunitTestRunner.cs new file mode 100644 index 00000000..752dc5c6 --- /dev/null +++ b/test/Shared/TestUtilities/Xunit/JetXunitTestRunner.cs @@ -0,0 +1,174 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit; + +public class JetXunitTestRunner : XunitTestRunner +{ + public JetXunitTestRunner( + ITest test, + IMessageBus messageBus, + Type testClass, + object[] constructorArguments, + MethodInfo testMethod, + object[] testMethodArguments, + string skipReason, + IReadOnlyList beforeAfterAttributes, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) + : base( + test, + messageBus, + testClass, + constructorArguments, + testMethod, + testMethodArguments, + skipReason, + beforeAfterAttributes, + aggregator, + cancellationTokenSource) + { + } + + public new async Task RunAsync() + { + var runSummary = new RunSummary { Total = 1 }; + var output = string.Empty; + + if (!MessageBus.QueueMessage(new TestStarting(Test))) + { + CancellationTokenSource.Cancel(); + } + else + { + AfterTestStarting(); + + if (!string.IsNullOrEmpty(SkipReason)) + { + ++runSummary.Skipped; + + if (!MessageBus.QueueMessage( + new TestSkipped(Test, SkipReason))) + { + CancellationTokenSource.Cancel(); + } + } + else + { + var aggregator = new ExceptionAggregator(Aggregator); + if (!aggregator.HasExceptions) + { + var tuple = await aggregator.RunAsync(() => InvokeTestAsync(aggregator)); + if (tuple != null) + { + runSummary.Time = tuple.Item1; + output = tuple.Item2; + } + } + + TestResultMessage testResultMessage; + + var exception = aggregator.ToException(); + if (exception == null) + { + testResultMessage = new TestPassed(Test, runSummary.Time, output); + } + + #region Customized + /// This is what we are after. Mark failed tests as 'Skipped', if the failure is expected. + else if (SkipFailedTest(exception)) + { + testResultMessage = new TestSkipped(Test, exception.Message); + ++runSummary.Skipped; + } + #endregion Customized + + else + { + testResultMessage = new TestFailed(Test, runSummary.Time, output, exception); + ++runSummary.Failed; + } + + if (!CancellationTokenSource.IsCancellationRequested && + !MessageBus.QueueMessage(testResultMessage)) + { + CancellationTokenSource.Cancel(); + } + } + + Aggregator.Clear(); + + BeforeTestFinished(); + + if (Aggregator.HasExceptions && !MessageBus.QueueMessage( + new TestCleanupFailure(Test, Aggregator.ToException()))) + { + CancellationTokenSource.Cancel(); + } + + if (!MessageBus.QueueMessage(new TestFinished(Test, runSummary.Time, output))) + { + CancellationTokenSource.Cancel(); + } + } + + return runSummary; + } + + /// + /// Mark failed tests as 'Skipped', if they failed because they use an expression, that we explicitly marked as + /// supported by Jet. + /// + protected virtual bool SkipFailedTest(Exception exception) + { + var skip = true; + var unexpectedUnsupportedTranslation = false; + + var aggregateException = exception as AggregateException ?? + new AggregateException(exception); + + foreach (var innerException in aggregateException.InnerExceptions) + { + if (!skip || + innerException is not InvalidOperationException) + { + return false; + } + + if (innerException.Message.StartsWith("Jet does not support ")) + { + var expectedUnsupportedTranslation = innerException.Message.Contains("APPLY statements") || + innerException.Message.Contains("skipping rows"); + + skip &= expectedUnsupportedTranslation; + unexpectedUnsupportedTranslation = !expectedUnsupportedTranslation; + } + else if (innerException.Message.StartsWith("The LINQ expression '") && + innerException.Message.Contains("' could not be translated.")) + { + var expectedUnsupportedTranslation = innerException.Message.Contains("RowNumberExpression"); + + skip &= expectedUnsupportedTranslation; + unexpectedUnsupportedTranslation = !expectedUnsupportedTranslation; + } + + if (unexpectedUnsupportedTranslation) + { + var sb = new StringBuilder(); + sb.AppendLine(innerException.Message.ReplaceLineEndings(" ")); + sb.AppendLine("-----"); + + File.AppendAllText("UnsupportedTranslations.txt", sb.ToString()); + } + } + + return skip; + } +} \ No newline at end of file diff --git a/test/Shared/TestUtilities/Xunit/JetXunitTheoryTestCaseRunner.cs b/test/Shared/TestUtilities/Xunit/JetXunitTheoryTestCaseRunner.cs new file mode 100644 index 00000000..de5428a1 --- /dev/null +++ b/test/Shared/TestUtilities/Xunit/JetXunitTheoryTestCaseRunner.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Threading; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit; + +public class JetXunitTheoryTestCaseRunner : XunitTheoryTestCaseRunner +{ + public JetXunitTheoryTestCaseRunner( + IXunitTestCase testCase, + string displayName, + string skipReason, + object[] constructorArguments, + IMessageSink diagnosticMessageSink, + IMessageBus messageBus, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) + : base( + testCase, + displayName, + skipReason, + constructorArguments, + diagnosticMessageSink, + messageBus, + aggregator, + cancellationTokenSource) + { + } + + protected override XunitTestRunner CreateTestRunner( + ITest test, + IMessageBus messageBus, + Type testClass, + object[] constructorArguments, + MethodInfo testMethod, + object[] testMethodArguments, + string skipReason, + IReadOnlyList beforeAfterAttributes, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) + => new JetXunitTestRunner( + test, + messageBus, + testClass, + constructorArguments, + testMethod, + testMethodArguments, + skipReason, + beforeAfterAttributes, + new ExceptionAggregator(aggregator), + cancellationTokenSource); +} \ No newline at end of file diff --git a/test/Shared/TestUtilities/Xunit/RandomTestCaseOrderer.cs b/test/Shared/TestUtilities/Xunit/RandomTestCaseOrderer.cs new file mode 100644 index 00000000..61bc2b8d --- /dev/null +++ b/test/Shared/TestUtilities/Xunit/RandomTestCaseOrderer.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using Xunit.Abstractions; +using Xunit.Sdk; + +// ReSharper disable once CheckNamespace +namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit; + +public class RandomTestCaseOrderer : ITestCaseOrderer +{ + public IEnumerable OrderTestCases(IEnumerable testCases) + where TTestCase : ITestCase + { + var random = new Random(); + var orderedTestCases = testCases.OrderBy(c => random.NextDouble()).ToList(); + + var builder = new StringBuilder() + .AppendLine("Test Case Order:") + .AppendLine(string.Join(Environment.NewLine, orderedTestCases.Select(c => c.TestMethod.Method.Name))); + + Debug.WriteLine(builder); + Console.WriteLine(builder); + + return orderedTestCases; + } +} \ No newline at end of file