From 9a6eb4dc80ed21820e92aef832da4c056ec073b2 Mon Sep 17 00:00:00 2001 From: Patrick Stevens <3138005+Smaug123@users.noreply.github.com> Date: Thu, 6 Jun 2024 00:09:09 +0100 Subject: [PATCH] Abstract out a Progress indicator (#28) --- TestRunner.Lib/Domain.fs | 19 ++++++ TestRunner.Lib/SurfaceBaseline.txt | 10 +++- TestRunner.Lib/TestFixture.fs | 36 ++++-------- TestRunner.Lib/TestProgress.fs | 45 ++++++++++++++ TestRunner.Lib/TestRunner.Lib.fsproj | 2 + TestRunner.Lib/version.json | 2 +- TestRunner/Program.fs | 87 +++++++++++++++------------- nix/deps.nix | 5 ++ 8 files changed, 139 insertions(+), 67 deletions(-) create mode 100644 TestRunner.Lib/TestProgress.fs diff --git a/TestRunner.Lib/Domain.fs b/TestRunner.Lib/Domain.fs index ad0cb2e..f9363e2 100644 --- a/TestRunner.Lib/Domain.fs +++ b/TestRunner.Lib/Domain.fs @@ -131,3 +131,22 @@ type TestFailure = | TestFailure.TestFailed f | TestFailure.SetUpFailed f | TestFailure.TearDownFailed f -> f.Name + +/// Represents the result of a test that didn't fail. +[] +type TestMemberSuccess = + /// The test passed. + | Ok + /// We didn't run the test, because it's []. + | Ignored of reason : string option + /// We didn't run the test, because it's []. + | Explicit of reason : string option + +/// Represents the failure of a test. +[] +type TestMemberFailure = + /// We couldn't run this test because it was somehow malformed in a way we detected up front. + | Malformed of reasons : string list + /// We tried to run the test, but it failed. (A single test can fail many times, e.g. if it failed and also + /// the tear-down logic failed afterwards.) + | Failed of TestFailure list diff --git a/TestRunner.Lib/SurfaceBaseline.txt b/TestRunner.Lib/SurfaceBaseline.txt index 5b62c34..e54c9c1 100644 --- a/TestRunner.Lib/SurfaceBaseline.txt +++ b/TestRunner.Lib/SurfaceBaseline.txt @@ -73,6 +73,12 @@ TestRunner.FixtureRunResults.get_OtherFailures [method]: unit -> TestRunner.User TestRunner.FixtureRunResults.get_SuccessCount [method]: unit -> int TestRunner.FixtureRunResults.OtherFailures [property]: [read-only] TestRunner.UserMethodFailure list TestRunner.FixtureRunResults.SuccessCount [property]: [read-only] int +TestRunner.ITestProgress - interface with 5 member(s) +TestRunner.ITestProgress.OnTestFailed [method]: string -> TestRunner.TestMemberFailure -> unit +TestRunner.ITestProgress.OnTestFixtureStart [method]: string -> int -> unit +TestRunner.ITestProgress.OnTestMemberFinished [method]: string -> unit +TestRunner.ITestProgress.OnTestMemberSkipped [method]: string -> unit +TestRunner.ITestProgress.OnTestMemberStart [method]: string -> unit TestRunner.Match inherit obj, implements TestRunner.Match System.IEquatable, System.Collections.IStructuralEquatable, TestRunner.Match System.IComparable, System.IComparable, System.Collections.IStructuralComparable - union type with 2 cases TestRunner.Match+Contains inherit TestRunner.Match TestRunner.Match+Contains.get_Item [method]: unit -> string @@ -171,7 +177,7 @@ TestRunner.TestFixture.TearDown [property]: [read-only] System.Reflection.Method TestRunner.TestFixture.Tests [property]: [read-only] TestRunner.SingleTestMethod list TestRunner.TestFixtureModule inherit obj TestRunner.TestFixtureModule.parse [static method]: System.Type -> TestRunner.TestFixture -TestRunner.TestFixtureModule.run [static method]: (TestRunner.TestFixture -> TestRunner.SingleTestMethod -> bool) -> TestRunner.TestFixture -> TestRunner.FixtureRunResults +TestRunner.TestFixtureModule.run [static method]: TestRunner.ITestProgress -> (TestRunner.TestFixture -> TestRunner.SingleTestMethod -> bool) -> TestRunner.TestFixture -> TestRunner.FixtureRunResults TestRunner.TestKind inherit obj, implements TestRunner.TestKind System.IEquatable, System.Collections.IStructuralEquatable - union type with 3 cases TestRunner.TestKind+Data inherit TestRunner.TestKind TestRunner.TestKind+Data.get_Item [method]: unit -> obj list list @@ -236,6 +242,8 @@ TestRunner.TestMemberSuccess.NewExplicit [static method]: string option -> TestR TestRunner.TestMemberSuccess.NewIgnored [static method]: string option -> TestRunner.TestMemberSuccess TestRunner.TestMemberSuccess.Ok [static property]: [read-only] TestRunner.TestMemberSuccess TestRunner.TestMemberSuccess.Tag [property]: [read-only] int +TestRunner.TestProgress inherit obj +TestRunner.TestProgress.toStderr [static method]: unit -> TestRunner.ITestProgress TestRunner.UserMethodFailure inherit obj, implements TestRunner.UserMethodFailure System.IEquatable, System.Collections.IStructuralEquatable - union type with 2 cases TestRunner.UserMethodFailure+ReturnedNonUnit inherit TestRunner.UserMethodFailure TestRunner.UserMethodFailure+ReturnedNonUnit.get_name [method]: unit -> string diff --git a/TestRunner.Lib/TestFixture.fs b/TestRunner.Lib/TestFixture.fs index 49f07b6..0b6694b 100644 --- a/TestRunner.Lib/TestFixture.fs +++ b/TestRunner.Lib/TestFixture.fs @@ -5,25 +5,6 @@ open System.Reflection open System.Threading open Microsoft.FSharp.Core -/// Represents the result of a test that didn't fail. -[] -type TestMemberSuccess = - /// The test passed. - | Ok - /// We didn't run the test, because it's []. - | Ignored of reason : string option - /// We didn't run the test, because it's []. - | Explicit of reason : string option - -/// Represents the failure of a test. -[] -type TestMemberFailure = - /// We couldn't run this test because it was somehow malformed in a way we detected up front. - | Malformed of reasons : string list - /// We tried to run the test, but it failed. (A single test can fail many times, e.g. if it failed and also - /// the tear-down logic failed afterwards.) - | Failed of TestFailure list - /// The results of running a single TestFixture. type FixtureRunResults = { @@ -269,8 +250,13 @@ module TestFixture = /// Run every test (except those which fail the `filter`) in this test fixture, as well as the /// appropriate setup and tear-down logic. - let run (filter : TestFixture -> SingleTestMethod -> bool) (tests : TestFixture) : FixtureRunResults = - eprintfn $"Running test fixture: %s{tests.Name} (%i{tests.Tests.Length} tests to run)" + let run + (progress : ITestProgress) + (filter : TestFixture -> SingleTestMethod -> bool) + (tests : TestFixture) + : FixtureRunResults + = + progress.OnTestFixtureStart tests.Name tests.Tests.Length let containingObject = let methods = @@ -316,7 +302,7 @@ module TestFixture = | None -> for test in tests.Tests do if filter tests test then - eprintfn $"Running test: %s{test.Name}" + progress.OnTestMemberStart test.Name let testSuccess = ref 0 let results = runTestsFromMember tests.SetUp tests.TearDown containingObject test @@ -325,13 +311,13 @@ module TestFixture = match result with | Error failure -> testFailures.Add failure - eprintfn $"Test failed: %O{failure}" + progress.OnTestFailed test.Name failure | Ok _ -> Interlocked.Increment testSuccess |> ignore Interlocked.Add (totalTestSuccess, testSuccess.Value) |> ignore - eprintfn $"Finished test %s{test.Name} (%i{testSuccess.Value} success)" + progress.OnTestMemberFinished test.Name else - eprintfn $"Skipping test due to filter: %s{test.Name}" + progress.OnTestMemberSkipped test.Name // Unconditionally run OneTimeTearDown if it exists. let tearDownError = diff --git a/TestRunner.Lib/TestProgress.fs b/TestRunner.Lib/TestProgress.fs new file mode 100644 index 0000000..a71972d --- /dev/null +++ b/TestRunner.Lib/TestProgress.fs @@ -0,0 +1,45 @@ +namespace TestRunner + +open System + +/// Represents something which knows how to report progress through a test suite. +/// Note that we don't guarantee anything about parallelism; you must make sure +/// all implementations are safe to run concurrently. +type ITestProgress = + /// Called just before we start executing the setup logic for the given test fixture. + /// We tell you how many test methods there are in the fixture. + abstract OnTestFixtureStart : name : string -> testCount : int -> unit + /// Called just before we start executing the test(s) indicated by a particular method. + abstract OnTestMemberStart : name : string -> unit + /// Called when a test fails. (This may be called repeatedly with the same `name`, e.g. if the test + /// is run multiple times with different combinations of test data.) + abstract OnTestFailed : name : string -> failure : TestMemberFailure -> unit + /// Called when we've finished every test indicated by a particular method. (The test may have been run + /// multiple times, e.g. with different combinations of test data.) + abstract OnTestMemberFinished : name : string -> unit + /// Called when we decide not to run the test(s) indicated by a particular method (e.g. because it's + /// marked []). + abstract OnTestMemberSkipped : name : string -> unit + +/// Methods for constructing specific ITestProgress objects. +[] +module TestProgress = + /// An ITestProgress which logs to stderr. + let toStderr () : ITestProgress = + { new ITestProgress with + member _.OnTestFixtureStart name testCount = + let plural = if testCount = 1 then "" else "s" + Console.Error.WriteLine $"Running test fixture: %s{name} (%i{testCount} test%s{plural} to run)" + + member _.OnTestMemberStart name = + Console.Error.WriteLine $"Running test: %s{name}" + + member _.OnTestFailed name failure = + Console.Error.WriteLine $"Test failed: %O{failure}" + + member _.OnTestMemberFinished name = + Console.Error.WriteLine $"Finished test %s{name}" + + member _.OnTestMemberSkipped name = + Console.Error.WriteLine $"Skipping test due to filter: %s{name}" + } diff --git a/TestRunner.Lib/TestRunner.Lib.fsproj b/TestRunner.Lib/TestRunner.Lib.fsproj index 54d770b..0c609bb 100644 --- a/TestRunner.Lib/TestRunner.Lib.fsproj +++ b/TestRunner.Lib/TestRunner.Lib.fsproj @@ -23,6 +23,7 @@ + True @@ -33,6 +34,7 @@ + diff --git a/TestRunner.Lib/version.json b/TestRunner.Lib/version.json index 0ff4bf6..f57433b 100644 --- a/TestRunner.Lib/version.json +++ b/TestRunner.Lib/version.json @@ -1,5 +1,5 @@ { - "version": "0.1", + "version": "0.2", "publicReleaseRefSpec": null, "pathFilters": [ "./", diff --git a/TestRunner/Program.fs b/TestRunner/Program.fs index 2e1d717..0f7b5a9 100644 --- a/TestRunner/Program.fs +++ b/TestRunner/Program.fs @@ -4,6 +4,17 @@ open System open System.IO open System.Reflection +// Fix for https://github.com/Smaug123/unofficial-nunit-runner/issues/8 +// Set AppContext.BaseDirectory to where the test DLL is. +// (This tells the DLL loader to look next to the test DLL for dependencies.) +type SetBaseDir (testDll : FileInfo) = + let oldBaseDir = AppContext.BaseDirectory + do AppContext.SetData ("APP_CONTEXT_BASE_DIRECTORY", testDll.Directory.FullName) + + interface IDisposable with + member _.Dispose () = + AppContext.SetData ("APP_CONTEXT_BASE_DIRECTORY", oldBaseDir) + module Program = let main argv = let testDll, filter = @@ -17,50 +28,46 @@ module Program = | Some filter -> Filter.shouldRun filter | None -> fun _ _ -> true - // Fix for https://github.com/Smaug123/unofficial-nunit-runner/issues/8 - // Set AppContext.BaseDirectory to where the test DLL is. - // (This tells the DLL loader to look next to the test DLL for dependencies.) - let oldBaseDir = AppContext.BaseDirectory - AppContext.SetData ("APP_CONTEXT_BASE_DIRECTORY", testDll.Directory.FullName) + let progress = TestProgress.toStderr () + + use _ = new SetBaseDir (testDll) + let assy = Assembly.LoadFrom testDll.FullName let anyFailures = - try - assy.ExportedTypes - // TODO: NUnit nowadays doesn't care if you're a TestFixture or not - |> Seq.filter (fun ty -> - ty.CustomAttributes - |> Seq.exists (fun attr -> attr.AttributeType.FullName = "NUnit.Framework.TestFixtureAttribute") + assy.ExportedTypes + // TODO: NUnit nowadays doesn't care if you're a TestFixture or not + |> Seq.filter (fun ty -> + ty.CustomAttributes + |> Seq.exists (fun attr -> attr.AttributeType.FullName = "NUnit.Framework.TestFixtureAttribute") + ) + |> Seq.fold + (fun anyFailures ty -> + let testFixture = TestFixture.parse ty + + let results = TestFixture.run progress filter testFixture + + let anyFailures = + match results.Failed with + | [] -> anyFailures + | _ :: _ -> + eprintfn $"%i{results.Failed.Length} tests failed" + true + + let anyFailures = + match results.OtherFailures with + | [] -> anyFailures + | otherFailures -> + eprintfn "Other failures encountered: " + + for failure in otherFailures do + eprintfn $" %s{failure.Name}" + + true + + anyFailures ) - |> Seq.fold - (fun anyFailures ty -> - let testFixture = TestFixture.parse ty - - let results = TestFixture.run filter testFixture - - let anyFailures = - match results.Failed with - | [] -> anyFailures - | _ :: _ -> - eprintfn $"%i{results.Failed.Length} tests failed" - true - - let anyFailures = - match results.OtherFailures with - | [] -> anyFailures - | otherFailures -> - eprintfn "Other failures encountered: " - - for failure in otherFailures do - eprintfn $" %s{failure.Name}" - - true - - anyFailures - ) - false - finally - AppContext.SetData ("APP_CONTEXT_BASE_DIRECTORY", oldBaseDir) + false if anyFailures then 1 else 0 diff --git a/nix/deps.nix b/nix/deps.nix index ee0dfaa..9320cea 100644 --- a/nix/deps.nix +++ b/nix/deps.nix @@ -21,6 +21,11 @@ version = "0.26.0"; sha256 = "0xgv5kvbwfdvcp6s8x7xagbbi4s3mqa4ixni6pazqvyflbgnah7b"; }) + (fetchNuGet { + pname = "FSharp.Core"; + version = "6.0.0"; + sha256 = "1hjhvr39c1vpgrdmf8xln5q86424fqkvy9nirkr29vl2461d2039"; + }) (fetchNuGet { pname = "FSharp.Core"; version = "8.0.300";