Experimental: Register dynamic tests at runtime from within other tests (#2186)

* Basic structure

* Register dynamic tests at runtime from within other tests

* Update Public API

* Update snaps

* Docs

Author: thomhurst

TUnit.Core/Attributes/RunOnDiscoveryAttribute.cs

@@ -0,0 +1,13 @@
+using TUnit.Core.Interfaces;
+
+namespace TUnit.Core;
+
+public class RunOnDiscoveryAttribute : TUnitAttribute, ITestDiscoveryEventReceiver
+{
+    public int Order => 0;
+
+    public void OnTestDiscovery(DiscoveredTestContext discoveredTestContext)
+    {
+        discoveredTestContext.RunOnTestDiscovery = true;
+    }
+}

TUnit.Core/DiscoveredTestContext.cs

@@ -21,17 +21,17 @@ public void AddProperty(string key, string value)
     {
         TestContext.TestDetails.InternalCustomProperties.Add(key, value);
     }
-    
+
     public void AddCategory(string category)
     {
         TestContext.TestDetails.MutableCategories.Add(category);
     }
-    
+
     public void SetDisplayName(string displayName)
     {
         TestContext.TestDetails.DisplayName = displayName;
     }
-    
+
     public void AddArgumentDisplayFormatter(ArgumentDisplayFormatter formatter)
     {
         TestContext.ArgumentDisplayFormatters.Add(formatter);
@@ -41,15 +41,21 @@ public void SetParallelConstraint(IParallelConstraint parallelConstraint)
     {
         TestContext.TestDetails.ParallelConstraint = parallelConstraint;
     }
-    
+
     public void SetRetryCount(int times)
     {
         SetRetryCount(times, (_, _, _) => Task.FromResult(true));
     }
-    
+
     public void SetRetryCount(int times, Func<TestContext, Exception, int, Task<bool>> shouldRetry)
     {
         TestContext.TestDetails.RetryLimit = times;
         TestContext.TestDetails.RetryLogic = shouldRetry;
     }
+
+    public bool RunOnTestDiscovery
+    {
+        get => TestContext.RunOnTestDiscovery;
+        set => TestContext.RunOnTestDiscovery = value;
+    }
 }

TUnit.Core/DynamicTestBuilderContext.cs

@@ -2,8 +2,21 @@
 
 namespace TUnit.Core;
 
-public class DynamicTestBuilderContext(string filePath, int lineNumber)
+public class DynamicTestBuilderContext
 {
+    private readonly string _filePath;
+    private readonly int _lineNumber;
+
+    public DynamicTestBuilderContext(string filePath, int lineNumber)
+    {
+        _filePath = filePath;
+        _lineNumber = lineNumber;
+    }
+
+    public DynamicTestBuilderContext(TestContext testContext) : this(testContext.TestDetails.TestFilePath, testContext.TestDetails.TestLineNumber)
+    {
+    }
+
     public List<DynamicTest> Tests { get; } = [];
 
     public void AddTest<
@@ -12,10 +25,21 @@ public void AddTest<
                                     | DynamicallyAccessedMemberTypes.PublicProperties)]
         TClass>(DynamicTest<TClass> dynamicTest) where TClass : class
     {
-        Tests.Add(dynamicTest with
+        var testToRegister = dynamicTest with
         {
-            TestFilePath = filePath,
-            TestLineNumber = lineNumber
-        });
+            TestFilePath = _filePath,
+            TestLineNumber = _lineNumber
+        };
+        
+        Tests.Add(testToRegister);
+    }
+
+    public async Task AddTestAtRuntime<
+        [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors 
+                                    | DynamicallyAccessedMemberTypes.PublicMethods
+                                    | DynamicallyAccessedMemberTypes.PublicProperties)]
+        TClass>(TestContext testContext, DynamicTest<TClass> dynamicTest) where TClass : class
+    {
+        await testContext.GetService<IDynamicTestRegistrar>().Register(dynamicTest);
     }
 }

TUnit.Core/Extensions/TestContextExtensions.cs

@@ -1,4 +1,5 @@
-using TUnit.Core.Helpers;
+using System.Diagnostics.CodeAnalysis;
+using TUnit.Core.Helpers;
 using TUnit.Core.Interfaces;
 
 namespace TUnit.Core.Extensions;
@@ -68,6 +69,16 @@ public static string GetClassTypeName(this TestContext testContext)
             $"{classTypeName}({string.Join(", ", testDetails.TestClassArguments.Select(x => ArgumentFormatter.GetConstantValue(testContext, x)))})";
     }
 
+    [Experimental("WIP")]
+    public static Task AddDynamicTest<
+        [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors 
+                                    | DynamicallyAccessedMemberTypes.PublicMethods
+                                    | DynamicallyAccessedMemberTypes.PublicProperties)]
+        T>(this TestContext testContext, DynamicTest<T> dynamicTest) where T : class
+    {
+        return new DynamicTestBuilderContext(testContext).AddTestAtRuntime(testContext, dynamicTest);
+    }
+    
     /// <summary>
     /// Gets the test display name for the test context.
     /// </summary>

TUnit.Core/IDynamicTestRegistrar.cs

@@ -0,0 +1,12 @@
+using System.Diagnostics.CodeAnalysis;
+
+namespace TUnit.Core;
+
+public interface IDynamicTestRegistrar
+{
+    Task Register<
+        [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors 
+                                    | DynamicallyAccessedMemberTypes.PublicMethods
+                                    | DynamicallyAccessedMemberTypes.PublicProperties)]
+        TClass>(DynamicTest<TClass> dynamicTest) where TClass : class;
+}

TUnit.Core/TestContext.cs

@@ -130,4 +130,6 @@ public void AddArtifact(Artifact artifact)
     /// Gets or sets the event objects.
     /// </summary>
     internal object?[]? EventObjects { get; set; }
+
+    internal bool RunOnTestDiscovery { get; set; }
 }

TUnit.Engine.Tests/DynamicTests.cs

@@ -0,0 +1,20 @@
+using Shouldly;
+
+namespace TUnit.Engine.Tests;
+
+public class DynamicTests : InvokableTestBase
+{
+    [Test]
+    public async Task Test()
+    {
+        await RunTestsWithFilter(
+            "/*/*DynamicTests/*/*",
+            [
+                result => result.ResultSummary.Outcome.ShouldBe("Completed"),
+                result => result.ResultSummary.Counters.Total.ShouldBe(70),
+                result => result.ResultSummary.Counters.Passed.ShouldBe(70),
+                result => result.ResultSummary.Counters.Failed.ShouldBe(0),
+                result => result.ResultSummary.Counters.NotExecuted.ShouldBe(0)
+            ]);
+    }
+}

TUnit.Engine/Extensions/TestContextExtensions.cs

@@ -24,6 +24,8 @@ public static async Task ReregisterTestWithArguments(
         object?[]? methodArguments, 
         Dictionary<string, object?>? objectBag = null)
     {
+        // TODO: Rework to use DynamicTestRegistrar
+        
         var testMetadata = testContext.OriginalMetadata;
 
         var testBuilderContext = new TestBuilderContext();
@@ -76,7 +78,7 @@ await testContext.GetService<TestRegistrar>().RegisterInstance(newTest,
             NotInParallel = new PriorityQueue<DiscoveredTest, int>(),
             KeyedNotInParallel = new Dictionary<ConstraintKeysCollection, PriorityQueue<DiscoveredTest, int>>(),
             ParallelGroups = new ConcurrentDictionary<ParallelGroupConstraint, List<DiscoveredTest>>()
-        }, null, testContext.GetService<ExecuteRequestContext>());
+        }, null, testContext.GetService<EngineCancellationToken>().CancellationTokenSource.Token);
     }
 
     internal static void SetResult(this TestContext testContext, Exception? exception)

TUnit.Engine/Framework/TUnitServiceProvider.cs

@@ -91,26 +91,32 @@ public TUnitServiceProvider(IExtension extension,
         var testHookOrchestrator = Register(new TestHookOrchestrator(HooksCollector, Logger));
 
         var testRegistrar = Register(new TestRegistrar(instanceTracker, AssemblyHookOrchestrator, classHookOrchestrator));
-        TestDiscoverer = Register(new TUnitTestDiscoverer(testsConstructor, testFilterService, TestGrouper, testRegistrar, TUnitMessageBus, Logger, extension));
-        
-        TestFinder = Register(new TestsFinder(TestDiscoverer));
-        Register<ITestFinder>(TestFinder);
 
         Disposer = Register(new Disposer(Logger));
 
         var testInvoker = Register(new TestInvoker(testHookOrchestrator, Logger, Disposer));
         var parallelLimitProvider = Register(new ParallelLimitLockProvider());
 
-        // TODO
-        Register(new HookMessagePublisher(extension, messageBus));
-        
         var singleTestExecutor = Register(new SingleTestExecutor(extension, instanceTracker, testInvoker, parallelLimitProvider, AssemblyHookOrchestrator, classHookOrchestrator, TUnitMessageBus, Logger, EngineCancellationToken, testRegistrar));
 
         TestsExecutor = Register(new TestsExecutor(singleTestExecutor, Logger, CommandLineOptions, EngineCancellationToken, AssemblyHookOrchestrator, classHookOrchestrator));
 
+        TestDiscoverer = Register(new TUnitTestDiscoverer(testsConstructor, testFilterService, TestGrouper, testRegistrar, TUnitMessageBus, Logger, TestsExecutor, extension));
+
+        DynamicTestRegistrar = Register<IDynamicTestRegistrar>(new DynamicTestRegistrar(testsConstructor, testRegistrar,
+            TestGrouper, TUnitMessageBus, TestsExecutor, EngineCancellationToken));
+        
+        TestFinder = Register(new TestsFinder(TestDiscoverer));
+        Register<ITestFinder>(TestFinder);
+        
+        // TODO
+        Register(new HookMessagePublisher(extension, messageBus));
+        
         OnEndExecutor = Register(new OnEndExecutor(CommandLineOptions, Logger));
     }
-    
+
+    public IDynamicTestRegistrar DynamicTestRegistrar { get; }
+
     public Disposer Disposer { get; }
 
     public async ValueTask DisposeAsync()

TUnit.Engine/Framework/TUnitTestFramework.cs

@@ -119,11 +119,13 @@ public async Task ExecuteRequestAsync(ExecuteRequestContext context)
                             TestFilter = stringFilter,
                             Id = runTestExecutionRequest.Session.SessionUid.Value
                         };
+                    
+                    TestSessionContext.Current = testSessionContext;
 
                     ExecutionContextHelper.RestoreContext(await serviceProvider.TestSessionHookOrchestrator.RunBeforeTestSession(context));
 
                     await serviceProvider.TestsExecutor.ExecuteAsync(filteredTests, runTestExecutionRequest.Filter,
-                        context);
+                        context.CancellationToken);
 
                     // Tests could reschedule separate invocations - This allows us to wait for all invocations
                     await serviceProvider.TestsExecutor.WaitForFinishAsync();