From df64e46079941a2de9594875c4e06af6f4bdf7c0 Mon Sep 17 00:00:00 2001 From: Patrick Stevens <3138005+Smaug123@users.noreply.github.com> Date: Tue, 11 Jun 2024 23:28:13 +0100 Subject: [PATCH] Cope with parameterised fixtures (#70) --- Consumer/Consumer.fsproj | 1 + Consumer/TestParameterisedFixture.fs | 18 +++ WoofWare.NUnitTestRunner.Lib/Domain.fs | 14 ++- .../SurfaceBaseline.txt | 12 +- WoofWare.NUnitTestRunner.Lib/TestFixture.fs | 107 ++++++++++++------ WoofWare.NUnitTestRunner.Lib/version.json | 2 +- WoofWare.NUnitTestRunner/Program.fs | 2 +- 7 files changed, 110 insertions(+), 46 deletions(-) create mode 100644 Consumer/TestParameterisedFixture.fs diff --git a/Consumer/Consumer.fsproj b/Consumer/Consumer.fsproj index 3db48ea..f482d4c 100644 --- a/Consumer/Consumer.fsproj +++ b/Consumer/Consumer.fsproj @@ -10,6 +10,7 @@ + diff --git a/Consumer/TestParameterisedFixture.fs b/Consumer/TestParameterisedFixture.fs new file mode 100644 index 0000000..babeaf5 --- /dev/null +++ b/Consumer/TestParameterisedFixture.fs @@ -0,0 +1,18 @@ +namespace Consumer + +open NUnit.Framework +open FsUnitTyped + +[] +[] +type TestParameterisedFixture (v : bool) = + [] + member _.Thing () = v |> shouldEqual v + +[] +[] +type TestParameterisedFixtureMultiple (i : int, v : bool) = + [] + member _.Thing () = + v |> shouldEqual v + i |> shouldEqual i diff --git a/WoofWare.NUnitTestRunner.Lib/Domain.fs b/WoofWare.NUnitTestRunner.Lib/Domain.fs index a832f8d..612bca8 100644 --- a/WoofWare.NUnitTestRunner.Lib/Domain.fs +++ b/WoofWare.NUnitTestRunner.Lib/Domain.fs @@ -1,5 +1,6 @@ namespace WoofWare.NUnitTestRunner +open System open System.Reflection /// A modifier on whether a given test should be run. @@ -62,6 +63,8 @@ type TestFixture = /// Fully-qualified name of this fixture (e.g. MyThing.Test.Foo for `[] module Foo` in the /// `MyThing.Test` assembly). Name : string + /// The type which is this fixture, containing the tests as members. + Type : Type /// A method which is run once when this test fixture starts, before any other setup logic and before /// any tests run. If this method fails, no tests will run and no per-test setup/teardown logic will run, /// but OneTimeTearDown will run. @@ -77,19 +80,24 @@ type TestFixture = /// Methods which are run in some arbitrary order after each individual test, even if the test or its setup /// failed. If the first TearDown we run fails, we don't define whether the other TearDowns run. TearDown : MethodInfo list + /// You might have defined e.g. `[] type Foo (v : bool) = ...`. If so, this gives the + /// various possible parameters. + Parameters : obj list list /// The individual test methods present within this fixture. Tests : SingleTestMethod list } /// A test fixture about which we know nothing. No tests, no setup/teardown. - static member Empty (containingAssembly : Assembly) (name : string) = + static member Empty (ty : Type) (args : obj list list) = { - ContainingAssembly = containingAssembly - Name = name + ContainingAssembly = ty.Assembly + Type = ty + Name = ty.Name OneTimeSetUp = None OneTimeTearDown = None SetUp = [] TearDown = [] + Parameters = args Tests = [] } diff --git a/WoofWare.NUnitTestRunner.Lib/SurfaceBaseline.txt b/WoofWare.NUnitTestRunner.Lib/SurfaceBaseline.txt index 651e6f8..0f1fffd 100644 --- a/WoofWare.NUnitTestRunner.Lib/SurfaceBaseline.txt +++ b/WoofWare.NUnitTestRunner.Lib/SurfaceBaseline.txt @@ -185,25 +185,29 @@ WoofWare.NUnitTestRunner.TestFailure.NewTearDownFailed [static method]: WoofWare WoofWare.NUnitTestRunner.TestFailure.NewTestFailed [static method]: WoofWare.NUnitTestRunner.UserMethodFailure -> WoofWare.NUnitTestRunner.TestFailure WoofWare.NUnitTestRunner.TestFailure.Tag [property]: [read-only] int WoofWare.NUnitTestRunner.TestFixture inherit obj, implements WoofWare.NUnitTestRunner.TestFixture System.IEquatable, System.Collections.IStructuralEquatable -WoofWare.NUnitTestRunner.TestFixture..ctor [constructor]: (System.Reflection.Assembly, string, System.Reflection.MethodInfo option, System.Reflection.MethodInfo option, System.Reflection.MethodInfo list, System.Reflection.MethodInfo list, WoofWare.NUnitTestRunner.SingleTestMethod list) +WoofWare.NUnitTestRunner.TestFixture..ctor [constructor]: (System.Reflection.Assembly, string, System.Type, System.Reflection.MethodInfo option, System.Reflection.MethodInfo option, System.Reflection.MethodInfo list, System.Reflection.MethodInfo list, obj list list, WoofWare.NUnitTestRunner.SingleTestMethod list) WoofWare.NUnitTestRunner.TestFixture.ContainingAssembly [property]: [read-only] System.Reflection.Assembly -WoofWare.NUnitTestRunner.TestFixture.Empty [static method]: System.Reflection.Assembly -> string -> WoofWare.NUnitTestRunner.TestFixture +WoofWare.NUnitTestRunner.TestFixture.Empty [static method]: System.Type -> obj list list -> WoofWare.NUnitTestRunner.TestFixture WoofWare.NUnitTestRunner.TestFixture.get_ContainingAssembly [method]: unit -> System.Reflection.Assembly WoofWare.NUnitTestRunner.TestFixture.get_Name [method]: unit -> string WoofWare.NUnitTestRunner.TestFixture.get_OneTimeSetUp [method]: unit -> System.Reflection.MethodInfo option WoofWare.NUnitTestRunner.TestFixture.get_OneTimeTearDown [method]: unit -> System.Reflection.MethodInfo option +WoofWare.NUnitTestRunner.TestFixture.get_Parameters [method]: unit -> obj list list WoofWare.NUnitTestRunner.TestFixture.get_SetUp [method]: unit -> System.Reflection.MethodInfo list WoofWare.NUnitTestRunner.TestFixture.get_TearDown [method]: unit -> System.Reflection.MethodInfo list WoofWare.NUnitTestRunner.TestFixture.get_Tests [method]: unit -> WoofWare.NUnitTestRunner.SingleTestMethod list +WoofWare.NUnitTestRunner.TestFixture.get_Type [method]: unit -> System.Type WoofWare.NUnitTestRunner.TestFixture.Name [property]: [read-only] string WoofWare.NUnitTestRunner.TestFixture.OneTimeSetUp [property]: [read-only] System.Reflection.MethodInfo option WoofWare.NUnitTestRunner.TestFixture.OneTimeTearDown [property]: [read-only] System.Reflection.MethodInfo option +WoofWare.NUnitTestRunner.TestFixture.Parameters [property]: [read-only] obj list list WoofWare.NUnitTestRunner.TestFixture.SetUp [property]: [read-only] System.Reflection.MethodInfo list WoofWare.NUnitTestRunner.TestFixture.TearDown [property]: [read-only] System.Reflection.MethodInfo list WoofWare.NUnitTestRunner.TestFixture.Tests [property]: [read-only] WoofWare.NUnitTestRunner.SingleTestMethod list +WoofWare.NUnitTestRunner.TestFixture.Type [property]: [read-only] System.Type WoofWare.NUnitTestRunner.TestFixtureModule inherit obj WoofWare.NUnitTestRunner.TestFixtureModule.parse [static method]: System.Type -> WoofWare.NUnitTestRunner.TestFixture -WoofWare.NUnitTestRunner.TestFixtureModule.run [static method]: WoofWare.NUnitTestRunner.ITestProgress -> (WoofWare.NUnitTestRunner.TestFixture -> WoofWare.NUnitTestRunner.SingleTestMethod -> bool) -> WoofWare.NUnitTestRunner.TestFixture -> WoofWare.NUnitTestRunner.FixtureRunResults +WoofWare.NUnitTestRunner.TestFixtureModule.run [static method]: WoofWare.NUnitTestRunner.ITestProgress -> (WoofWare.NUnitTestRunner.TestFixture -> WoofWare.NUnitTestRunner.SingleTestMethod -> bool) -> WoofWare.NUnitTestRunner.TestFixture -> WoofWare.NUnitTestRunner.FixtureRunResults list WoofWare.NUnitTestRunner.TestKind inherit obj, implements WoofWare.NUnitTestRunner.TestKind System.IEquatable, System.Collections.IStructuralEquatable - union type with 3 cases WoofWare.NUnitTestRunner.TestKind+Data inherit WoofWare.NUnitTestRunner.TestKind WoofWare.NUnitTestRunner.TestKind+Data.get_Item [method]: unit -> obj list list @@ -528,4 +532,4 @@ WoofWare.NUnitTestRunner.UserMethodFailure.IsThrew [property]: [read-only] bool WoofWare.NUnitTestRunner.UserMethodFailure.Name [property]: [read-only] string WoofWare.NUnitTestRunner.UserMethodFailure.NewReturnedNonUnit [static method]: (string, obj) -> WoofWare.NUnitTestRunner.UserMethodFailure WoofWare.NUnitTestRunner.UserMethodFailure.NewThrew [static method]: (string, System.Exception) -> WoofWare.NUnitTestRunner.UserMethodFailure -WoofWare.NUnitTestRunner.UserMethodFailure.Tag [property]: [read-only] int \ No newline at end of file +WoofWare.NUnitTestRunner.UserMethodFailure.Tag [property]: [read-only] int diff --git a/WoofWare.NUnitTestRunner.Lib/TestFixture.fs b/WoofWare.NUnitTestRunner.Lib/TestFixture.fs index dc63eac..dd257b7 100644 --- a/WoofWare.NUnitTestRunner.Lib/TestFixture.fs +++ b/WoofWare.NUnitTestRunner.Lib/TestFixture.fs @@ -1,6 +1,7 @@ namespace WoofWare.NUnitTestRunner open System +open System.Collections open System.Diagnostics open System.IO open System.Reflection @@ -415,36 +416,15 @@ module TestFixture = /// Run every test (except those which fail the `filter`) in this test fixture, as well as the /// appropriate setup and tear-down logic. - let run + let private runOneFixture (progress : ITestProgress) (filter : TestFixture -> SingleTestMethod -> bool) + (name : string) + (containingObject : obj) (tests : TestFixture) : FixtureRunResults = - progress.OnTestFixtureStart tests.Name tests.Tests.Length - - let containingObject = - let methods = - seq { - match tests.OneTimeSetUp with - | None -> () - | Some t -> yield t - - match tests.OneTimeTearDown with - | None -> () - | Some t -> yield t - - yield! tests.Tests |> Seq.map (fun t -> t.Method) - } - - methods - |> Seq.tryPick (fun mi -> - if not mi.IsStatic then - Some (Activator.CreateInstance mi.DeclaringType) - else - None - ) - |> Option.toObj + progress.OnTestFixtureStart name tests.Tests.Length let oldWorkDir = Environment.CurrentDirectory Environment.CurrentDirectory <- FileInfo(tests.ContainingAssembly.Location).Directory.FullName @@ -470,7 +450,7 @@ module TestFixture = ExecutionId = Guid.NewGuid () TestId = Guid.NewGuid () // This one is a bit dubious, because we don't actually have a test name at all - TestName = tests.Name + TestName = name ClassName = tests.Name StdOut = if String.IsNullOrEmpty stdOut then None else Some stdOut StdErr = if String.IsNullOrEmpty stdErr then None else Some stdErr @@ -540,19 +520,27 @@ module TestFixture = /// Interpret this type as a [], extracting the test members from it and annotating them with all /// relevant information about how we should run them. let parse (parentType : Type) : TestFixture = - if - parentType.CustomAttributes - |> Seq.exists (fun attr -> attr.AttributeType.FullName = "NUnit.Framework.SetUpFixtureAttribute") - then - failwith "This test runner does not support SetUpFixture. Please shout if you want this." + let categories, args = + (([], []), parentType.CustomAttributes) + ||> Seq.fold (fun (categories, args) attr -> + match attr.AttributeType.FullName with + | "NUnit.Framework.SetUpFixtureAttribute" -> + failwith "This test runner does not support SetUpFixture. Please shout if you want this." + | "NUnit.Framework.CategoryAttribute" -> + let cat = attr.ConstructorArguments |> Seq.exactlyOne |> _.Value |> unbox + cat :: categories, args + | "NUnit.Framework.TestFixtureAttribute" -> + let newArgs = + match attr.ConstructorArguments |> Seq.map _.Value |> Seq.toList with + | [ :? ICollection as x ] -> + x |> Seq.cast |> Seq.map _.Value |> Seq.toList + | xs -> xs - let categories = - parentType.CustomAttributes - |> Seq.filter (fun attr -> attr.AttributeType.FullName = "NUnit.Framework.CategoryAttribute") - |> Seq.map (fun attr -> attr.ConstructorArguments |> Seq.exactlyOne |> _.Value |> unbox) - |> Seq.toList + categories, newArgs :: args + | _ -> categories, args + ) - (TestFixture.Empty parentType.Assembly parentType.Name, parentType.GetRuntimeMethods ()) + (TestFixture.Empty parentType args, parentType.GetRuntimeMethods ()) ||> Seq.fold (fun state mi -> ((state, []), mi.CustomAttributes) ||> Seq.fold (fun (state, unrecognisedAttrs) attr -> @@ -622,3 +610,48 @@ module TestFixture = state ) + + /// Run every test (except those which fail the `filter`) in this test fixture, as well as the + /// appropriate setup and tear-down logic. + let run + (progress : ITestProgress) + (filter : TestFixture -> SingleTestMethod -> bool) + (tests : TestFixture) + : FixtureRunResults list + = + match tests.Parameters with + | [] -> [ null ] + | args -> args |> List.map List.toArray + |> List.map (fun args -> + let containingObject = + let methods = + seq { + match tests.OneTimeSetUp with + | None -> () + | Some t -> yield t + + match tests.OneTimeTearDown with + | None -> () + | Some t -> yield t + + yield! tests.Tests |> Seq.map (fun t -> t.Method) + } + + methods + |> Seq.tryPick (fun mi -> + if not mi.IsStatic then + Some (Activator.CreateInstance (mi.DeclaringType, args)) + else + None + ) + |> Option.toObj + + let name = + if isNull args then + tests.Name + else + let args = args |> Seq.map (fun o -> o.ToString ()) |> String.concat "," + $"%s{tests.Name}(%s{args})" + + runOneFixture progress filter name containingObject tests + ) diff --git a/WoofWare.NUnitTestRunner.Lib/version.json b/WoofWare.NUnitTestRunner.Lib/version.json index 944befb..57ef466 100644 --- a/WoofWare.NUnitTestRunner.Lib/version.json +++ b/WoofWare.NUnitTestRunner.Lib/version.json @@ -1,5 +1,5 @@ { - "version": "0.8", + "version": "0.9", "publicReleaseRefSpec": [ "^refs/heads/main$" ], diff --git a/WoofWare.NUnitTestRunner/Program.fs b/WoofWare.NUnitTestRunner/Program.fs index dfd2919..b9b2584 100644 --- a/WoofWare.NUnitTestRunner/Program.fs +++ b/WoofWare.NUnitTestRunner/Program.fs @@ -167,7 +167,7 @@ module Program = let testFixtures = assy.ExportedTypes |> Seq.map TestFixture.parse |> Seq.toList let creationTime = DateTimeOffset.Now - let results = testFixtures |> List.map (TestFixture.run progress filter) + let results = testFixtures |> List.collect (TestFixture.run progress filter) let finishTime = DateTimeOffset.Now let finishTimeHumanReadable = finishTime.ToString @"yyyy-MM-dd HH:mm:ss"