Finalize implementation and add crash detection support.

Author: lauxjpn

test/EFCore.Jet.FunctionalTests/EFCore.Jet.FunctionalTests.csproj

@@ -8,9 +8,11 @@
 	</PropertyGroup>
 
 	<ItemGroup>
-		<Compile Include="..\Shared\**\*.cs" />
+		<Compile Include="..\Shared\**\*.cs">
+			<Link>%(RecursiveDir)%(Filename)%(Extension)</Link>
+		</Compile>
 	</ItemGroup>
-
+	
 	<ItemGroup>
 		<PackageReference Include="MSTest.TestFramework" />
 		<PackageReference Include="xunit.core" />

test/EFCore.Jet.FunctionalTests/Properties/AssemblyInfo.cs

@@ -15,4 +15,4 @@
 
 #endif
 
-[assembly: TestFramework("EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit.JetXunitTestFramework", "EntityFrameworkCore.Jet.FunctionalTests")]
+[assembly: TestFramework("EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit." + nameof(JetXunitTestFramework), "EntityFrameworkCore.Jet.FunctionalTests")]

test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetXunitTestCaseRunner.cs

@@ -7,10 +7,6 @@
 		<Platforms>AnyCPU;x86</Platforms>
 	</PropertyGroup>
 
-	<ItemGroup>
-		<Compile Include="..\Shared\**\*.cs" />
-	</ItemGroup>
-
 	<ItemGroup>
 		<None Update="config.json">
 			<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>

test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/JetXunitTestFramework.cs

@@ -0,0 +1,14 @@
+using System;
+
+namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities;
+
+[Flags]
+public enum AccessProviderTypeVariation
+{
+    None = 0,
+    X86 = 1 << 0,
+    X64 = 1 << 1,
+    Odbc = 1 << 2,
+    OleDb = 1 << 3,
+    All = -1,
+}

test/EFCore.Jet.Tests/EFCore.Jet.Tests.csproj

@@ -0,0 +1,43 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.EntityFrameworkCore.TestUtilities.Xunit;
+
+namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities;
+
+/// <summary>
+/// Marks a test method or class that is known to crash the test runner.
+/// </summary>
+[AttributeUsage(AttributeTargets.Class | AttributeTargets.Method, AllowMultiple = true)]
+public class TestRunnerCrashAttribute : Attribute, ITestCondition
+{
+    public const string DefaultSkipReason = "The test is known to crash the test runner.";
+    
+    protected AccessProviderTypeVariation[] AccessProviderTypeVariations { get; }
+
+    public TestRunnerCrashAttribute(params AccessProviderTypeVariation[] accessProviderTypeVariations)
+    {
+        AccessProviderTypeVariations = accessProviderTypeVariations.Length > 0
+            ? accessProviderTypeVariations
+            : new[] { AccessProviderTypeVariation.All };
+    }
+
+    public virtual ValueTask<bool> IsMetAsync()
+    {
+        // Implement and enable if we want to filter tests by specific runtime scenarios.
+        var currentVariation = AccessProviderTypeVariation.All; // AppConfig.AccessProviderTypeVariation;
+        var isMet = AccessProviderTypeVariations.Any(v => v.HasFlag(currentVariation));
+
+        if (!isMet && string.IsNullOrEmpty(Skip))
+        {
+            Skip = DefaultSkipReason;
+        }
+
+        return new ValueTask<bool>(isMet);
+    }
+
+    public virtual string SkipReason
+        => Skip;
+
+    public virtual string Skip { get; set; }
+}

test/Shared/TestUtilities/Attributes/AccessProviderTypeVariation.cs

@@ -0,0 +1,135 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Reflection;
+using System.Threading;
+using System.Threading.Tasks;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit;
+
+public class JetXunitTestCaseRunner : XunitTestCaseRunner
+{
+    public const string TestRunnerCrashCacheDirectory = "TestRunnerCrashCache";
+    public const string AutoSkipPrefix = "[AutoSkip]";
+    public const string AutoSkipTestRunnerCrashingTestsEnvironmentVariableName = "EFCoreJet_AutoSkipTestRunnerCrashingTests";
+
+    public virtual bool EnableAutoSkipTestsKnownToCrashTestRunner
+        => (Environment.GetEnvironmentVariable(AutoSkipTestRunnerCrashingTestsEnvironmentVariableName)?.ToLowerInvariant() ?? "true") != "false";
+
+    public JetXunitTestCaseRunner(IXunitTestCase testCase,
+        string displayName,
+        string skipReason,
+        object[] constructorArguments,
+        object[] testMethodArguments,
+        IMessageBus messageBus,
+        ExceptionAggregator aggregator,
+        CancellationTokenSource cancellationTokenSource)
+        : base(
+            testCase,
+            displayName,
+            skipReason,
+            constructorArguments,
+            testMethodArguments,
+            messageBus,
+            aggregator,
+            cancellationTokenSource)
+    {
+    }
+
+    protected override XunitTestRunner CreateTestRunner(ITest test,
+        IMessageBus messageBus,
+        Type testClass,
+        object[] constructorArguments,
+        MethodInfo testMethod,
+        object[] testMethodArguments,
+        string skipReason,
+        IReadOnlyList<BeforeAfterTestAttribute> beforeAfterAttributes,
+        ExceptionAggregator aggregator,
+        CancellationTokenSource cancellationTokenSource)
+        => new JetXunitTestRunner(
+            test,
+            messageBus,
+            testClass,
+            constructorArguments,
+            testMethod,
+            testMethodArguments,
+            skipReason,
+            beforeAfterAttributes,
+            new ExceptionAggregator(aggregator),
+            cancellationTokenSource);
+
+    /// <remarks>
+    /// `TestRunner&lt;TTestCase&gt;.RunAsync()` is not virtual, so we need to override this method here to call our own
+    /// `JetXunitTestRunner.RunAsync()` implementation.
+    /// </remarks>>
+    protected override async Task<RunSummary> RunTestAsync()
+    {
+        if (EnableAutoSkipTestsKnownToCrashTestRunner)
+        {
+            AutoSkipTestsKnownToCrashTestRunner();
+        }
+
+        return await RunWithCrashDetection(
+            () => ((JetXunitTestRunner)CreateTestRunner(
+                    CreateTest(TestCase, DisplayName),
+                    MessageBus,
+                    TestClass,
+                    ConstructorArguments,
+                    TestMethod,
+                    TestMethodArguments,
+                    SkipReason,
+                    BeforeAfterAttributes,
+                    Aggregator,
+                    CancellationTokenSource))
+                .RunAsync());
+    }
+
+    protected virtual async Task<RunSummary> RunWithCrashDetection(Func<Task<RunSummary>> func)
+    {
+        Directory.CreateDirectory(TestRunnerCrashCacheDirectory);
+
+        var filePath = Path.Combine(TestRunnerCrashCacheDirectory, $"{DateTime.UtcNow:yyyyMMdd'_'HHmmss.fffffff}_{(Environment.Is64BitProcess ? "x64" : "x86")}_{Guid.NewGuid()}.txt");
+        var contents = $"{TestCase.TestMethod.TestClass.Class.Name}\t{TestCase.TestMethod.Method.Name}";
+        await File.WriteAllTextAsync(filePath, contents);
+
+        var result = await func();
+
+        File.Delete(filePath);
+
+        return result;
+    }
+
+    protected virtual void AutoSkipTestsKnownToCrashTestRunner()
+    {
+        if (IsTestKnownToCrashTestRunner(TestCase))
+        {
+            SkipReason = $"{AutoSkipPrefix} {TestRunnerCrashAttribute.DefaultSkipReason}";
+        }
+    }
+
+    protected virtual bool IsTestKnownToCrashTestRunner(ITestCase testCase)
+    {
+        if (File.Exists(JetXunitTestFramework.TestsKnownToCrashTestRunnerFilePath))
+        {
+            foreach (var line in File.ReadLines(JetXunitTestFramework.TestsKnownToCrashTestRunnerFilePath))
+            {
+                var parts = line.Split('\t');
+                if (parts.Length >= 2)
+                {
+                    var testClass = parts[^2];
+                    var testMethod = parts[^1];
+
+                    if (testClass == testCase.TestMethod.TestClass.Class.Name &&
+                        testMethod == testCase.TestMethod.Method.Name)
+                    {
+                        return true;
+                    }
+                }
+            }
+        }
+
+        return false;
+    }
+}

test/Shared/TestUtilities/Attributes/TestRunnerCrashAttribute.cs

@@ -0,0 +1,44 @@
+using System;
+using System.IO;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace EntityFrameworkCore.Jet.FunctionalTests.TestUtilities.Xunit;
+
+public class JetXunitTestFramework : XunitTestFramework
+{
+    public const string TestsKnownToCrashTestRunnerFilePath = "./../../../TestsKnownToCrashTestRunner.txt";
+    public const string DetectCrashesOfPreviousRunsEnvironmentVariableName = "EFCoreJet_DetectCrashesOfPreviousRuns";
+
+    public virtual bool EnableDetectCrashesOfPreviousRuns
+        => Environment.GetEnvironmentVariable(DetectCrashesOfPreviousRunsEnvironmentVariableName)?.ToLowerInvariant() == "true";
+
+    public JetXunitTestFramework(IMessageSink messageSink) : base(messageSink)
+    {
+    }
+
+    protected override ITestFrameworkDiscoverer CreateDiscoverer(IAssemblyInfo assemblyInfo)
+    {
+        if (EnableDetectCrashesOfPreviousRuns)
+        {
+            DetectCrashesOfPreviousRuns();
+        }
+        
+        return new JetXunitTestFrameworkDiscoverer(assemblyInfo, SourceInformationProvider, DiagnosticMessageSink);
+    }
+
+    protected virtual void DetectCrashesOfPreviousRuns()
+    {
+        if (!Directory.Exists(JetXunitTestCaseRunner.TestRunnerCrashCacheDirectory))
+            return;
+        
+        foreach (var filePath in Directory.EnumerateFiles(JetXunitTestCaseRunner.TestRunnerCrashCacheDirectory, "*.txt"))
+        {
+            var contents = File.ReadAllText(filePath);
+            contents = $"{string.Join('\t', Path.GetFileNameWithoutExtension(filePath).Split('_'))}\t{contents}\n";
+            File.AppendAllText(TestsKnownToCrashTestRunnerFilePath, contents);
+            
+            File.Delete(filePath);
+        }
+    }
+}

test/EFCore.Jet.FunctionalTests/TestUtilities/Xunit/AscendingTestCaseOrderer.cs renamed to test/Shared/TestUtilities/Xunit/AscendingTestCaseOrderer.cs

@@ -27,9 +27,8 @@ public JetXunitTestFrameworkDiscoverer(
     }
 
     protected override bool IsValidTestClass(ITypeInfo type)
-        => base.IsValidTestClass(type) /* &&
-           IsTestConditionMet<SupportedServerVersionConditionAttribute>(type) &&
-           IsTestConditionMet<SupportedServerVersionLessThanConditionAttribute>(type)*/;
+        => base.IsValidTestClass(type) &&
+           IsTestConditionMet<TestRunnerCrashAttribute>(type);
 
     protected virtual bool IsTestConditionMet<TType>(ITypeInfo type) where TType : ITestCondition
         => GetTestConditions<TType>(type).Aggregate(true, (current, next) => current && next.IsMetAsync().Result);