From d1fa66a2e8dec328d642bb5119f9e8df8c93eca7 Mon Sep 17 00:00:00 2001 From: Patrick Stevens <3138005+Smaug123@users.noreply.github.com> Date: Tue, 25 Mar 2025 21:49:50 +0000 Subject: [PATCH] Fix behaviour of `Explicit` test fixtures (#244) --- Consumer/Consumer.fsproj | 1 + Consumer/TestExplicit.fs | 24 +++++++ WoofWare.NUnitTestRunner.Lib/Domain.fs | 10 ++- .../SurfaceBaseline.txt | 9 ++- WoofWare.NUnitTestRunner.Lib/TestFixture.fs | 64 +++++++++++++++---- WoofWare.NUnitTestRunner.Lib/TestProgress.fs | 5 ++ WoofWare.NUnitTestRunner.Lib/version.json | 2 +- WoofWare.NUnitTestRunner/Progress.fs | 4 ++ 8 files changed, 100 insertions(+), 19 deletions(-) create mode 100644 Consumer/TestExplicit.fs diff --git a/Consumer/Consumer.fsproj b/Consumer/Consumer.fsproj index 81e1242..519521e 100644 --- a/Consumer/Consumer.fsproj +++ b/Consumer/Consumer.fsproj @@ -10,6 +10,7 @@ + diff --git a/Consumer/TestExplicit.fs b/Consumer/TestExplicit.fs new file mode 100644 index 0000000..c1f24c6 --- /dev/null +++ b/Consumer/TestExplicit.fs @@ -0,0 +1,24 @@ +namespace Consumer + +open NUnit.Framework + +[] +module TestExplicitIndividual = + + [] + [] + let ``This test should not be run`` () = failwith "should not call" + +[] +[] +module TestExplicitModule = + + [] + let setUp () = failwith "should not call: setup" + + [] + let tearDown () = + failwith "should not call: teardown" + + [] + let ``This test should not be run because its module is explicit`` () = failwith "should not call: test" diff --git a/WoofWare.NUnitTestRunner.Lib/Domain.fs b/WoofWare.NUnitTestRunner.Lib/Domain.fs index 22d1b51..10200e4 100644 --- a/WoofWare.NUnitTestRunner.Lib/Domain.fs +++ b/WoofWare.NUnitTestRunner.Lib/Domain.fs @@ -138,10 +138,17 @@ type TestFixture = Tests : SingleTestMethod list /// If this fixture has declared a parallelisability, that goes here. Parallelize : Parallelizable option + /// It is possible to mark a fixture as "Explicit" or "Ignored", for example. + Modifiers : Modifier list } /// A test fixture about which we know nothing. No tests, no setup/teardown. - static member Empty (ty : Type) (par : Parallelizable option) (args : obj list list) = + static member Empty + (ty : Type) + (par : Parallelizable option) + (modifiers : Modifier list) + (args : obj list list) + = { ContainingAssembly = ty.Assembly Type = ty @@ -153,6 +160,7 @@ type TestFixture = Parameters = args Tests = [] Parallelize = par + Modifiers = modifiers } /// User code in the unit under test has failed somehow. diff --git a/WoofWare.NUnitTestRunner.Lib/SurfaceBaseline.txt b/WoofWare.NUnitTestRunner.Lib/SurfaceBaseline.txt index 1ba69f1..2791eb8 100644 --- a/WoofWare.NUnitTestRunner.Lib/SurfaceBaseline.txt +++ b/WoofWare.NUnitTestRunner.Lib/SurfaceBaseline.txt @@ -170,8 +170,9 @@ WoofWare.NUnitTestRunner.IndividualTestRunMetadata.StdOut [property]: [read-only WoofWare.NUnitTestRunner.IndividualTestRunMetadata.TestId [property]: [read-only] System.Guid WoofWare.NUnitTestRunner.IndividualTestRunMetadata.TestName [property]: [read-only] string WoofWare.NUnitTestRunner.IndividualTestRunMetadata.Total [property]: [read-only] System.TimeSpan -WoofWare.NUnitTestRunner.ITestProgress - interface with 5 member(s) +WoofWare.NUnitTestRunner.ITestProgress - interface with 6 member(s) WoofWare.NUnitTestRunner.ITestProgress.OnTestFailed [method]: string -> WoofWare.NUnitTestRunner.TestMemberFailure -> unit +WoofWare.NUnitTestRunner.ITestProgress.OnTestFixtureSkipped [method]: string -> string -> unit WoofWare.NUnitTestRunner.ITestProgress.OnTestFixtureStart [method]: string -> int -> unit WoofWare.NUnitTestRunner.ITestProgress.OnTestMemberFinished [method]: string -> unit WoofWare.NUnitTestRunner.ITestProgress.OnTestMemberSkipped [method]: string -> unit @@ -346,11 +347,12 @@ WoofWare.NUnitTestRunner.TestFailure.NewTearDownFailed [static method]: WoofWare WoofWare.NUnitTestRunner.TestFailure.NewTestFailed [static method]: WoofWare.NUnitTestRunner.UserMethodFailure -> WoofWare.NUnitTestRunner.TestFailure WoofWare.NUnitTestRunner.TestFailure.Tag [property]: [read-only] int WoofWare.NUnitTestRunner.TestFixture inherit obj, implements WoofWare.NUnitTestRunner.TestFixture System.IEquatable, System.Collections.IStructuralEquatable -WoofWare.NUnitTestRunner.TestFixture..ctor [constructor]: (System.Reflection.Assembly, string, System.Type, System.Reflection.MethodInfo option, System.Reflection.MethodInfo option, System.Reflection.MethodInfo list, System.Reflection.MethodInfo list, obj list list, WoofWare.NUnitTestRunner.SingleTestMethod list, WoofWare.NUnitTestRunner.ClassParallelScope WoofWare.NUnitTestRunner.Parallelizable option) +WoofWare.NUnitTestRunner.TestFixture..ctor [constructor]: (System.Reflection.Assembly, string, System.Type, System.Reflection.MethodInfo option, System.Reflection.MethodInfo option, System.Reflection.MethodInfo list, System.Reflection.MethodInfo list, obj list list, WoofWare.NUnitTestRunner.SingleTestMethod list, WoofWare.NUnitTestRunner.ClassParallelScope WoofWare.NUnitTestRunner.Parallelizable option, WoofWare.NUnitTestRunner.Modifier list) WoofWare.NUnitTestRunner.TestFixture.ContainingAssembly [property]: [read-only] System.Reflection.Assembly -WoofWare.NUnitTestRunner.TestFixture.Empty [static method]: System.Type -> WoofWare.NUnitTestRunner.ClassParallelScope WoofWare.NUnitTestRunner.Parallelizable option -> obj list list -> WoofWare.NUnitTestRunner.TestFixture +WoofWare.NUnitTestRunner.TestFixture.Empty [static method]: System.Type -> WoofWare.NUnitTestRunner.ClassParallelScope WoofWare.NUnitTestRunner.Parallelizable option -> WoofWare.NUnitTestRunner.Modifier list -> obj list list -> WoofWare.NUnitTestRunner.TestFixture WoofWare.NUnitTestRunner.TestFixture.Equals [method]: (WoofWare.NUnitTestRunner.TestFixture, System.Collections.IEqualityComparer) -> bool WoofWare.NUnitTestRunner.TestFixture.get_ContainingAssembly [method]: unit -> System.Reflection.Assembly +WoofWare.NUnitTestRunner.TestFixture.get_Modifiers [method]: unit -> WoofWare.NUnitTestRunner.Modifier list WoofWare.NUnitTestRunner.TestFixture.get_Name [method]: unit -> string WoofWare.NUnitTestRunner.TestFixture.get_OneTimeSetUp [method]: unit -> System.Reflection.MethodInfo option WoofWare.NUnitTestRunner.TestFixture.get_OneTimeTearDown [method]: unit -> System.Reflection.MethodInfo option @@ -360,6 +362,7 @@ WoofWare.NUnitTestRunner.TestFixture.get_SetUp [method]: unit -> System.Reflecti WoofWare.NUnitTestRunner.TestFixture.get_TearDown [method]: unit -> System.Reflection.MethodInfo list WoofWare.NUnitTestRunner.TestFixture.get_Tests [method]: unit -> WoofWare.NUnitTestRunner.SingleTestMethod list WoofWare.NUnitTestRunner.TestFixture.get_Type [method]: unit -> System.Type +WoofWare.NUnitTestRunner.TestFixture.Modifiers [property]: [read-only] WoofWare.NUnitTestRunner.Modifier list WoofWare.NUnitTestRunner.TestFixture.Name [property]: [read-only] string WoofWare.NUnitTestRunner.TestFixture.OneTimeSetUp [property]: [read-only] System.Reflection.MethodInfo option WoofWare.NUnitTestRunner.TestFixture.OneTimeTearDown [property]: [read-only] System.Reflection.MethodInfo option diff --git a/WoofWare.NUnitTestRunner.Lib/TestFixture.fs b/WoofWare.NUnitTestRunner.Lib/TestFixture.fs index 9dc13dd..3260b39 100644 --- a/WoofWare.NUnitTestRunner.Lib/TestFixture.fs +++ b/WoofWare.NUnitTestRunner.Lib/TestFixture.fs @@ -597,15 +597,15 @@ module TestFixture = /// Interpret this type as a [], extracting the test members from it and annotating them with all /// relevant information about how we should run them. let parse (parentType : Type) : TestFixture = - let categories, args, par = - (([], [], None), parentType.CustomAttributes) - ||> Seq.fold (fun (categories, args, par) attr -> + let categories, args, mods, par = + (([], [], [], None), parentType.CustomAttributes) + ||> Seq.fold (fun (categories, args, mods, par) attr -> match attr.AttributeType.FullName with | "NUnit.Framework.SetUpFixtureAttribute" -> failwith "This test runner does not support SetUpFixture. Please shout if you want this." | "NUnit.Framework.CategoryAttribute" -> let cat = attr.ConstructorArguments |> Seq.exactlyOne |> _.Value |> unbox - cat :: categories, args, par + cat :: categories, args, mods, par | "NUnit.Framework.TestFixtureAttribute" -> let newArgs = match attr.ConstructorArguments |> Seq.map _.Value |> Seq.toList with @@ -613,38 +613,52 @@ module TestFixture = x |> Seq.cast |> Seq.map _.Value |> Seq.toList | xs -> xs - categories, newArgs :: args, par + categories, newArgs :: args, mods, par | "NUnit.Framework.NonParallelizableAttribute" -> match par with | Some _ -> failwith $"Got multiple parallelism attributes on %s{parentType.FullName}" - | None -> categories, args, Some Parallelizable.No + | None -> categories, args, mods, Some Parallelizable.No | "NUnit.Framework.ParallelizableAttribute" -> match par with | Some _ -> failwith $"Got multiple parallelism attributes on %s{parentType.FullName}" | None -> match attr.ConstructorArguments |> Seq.toList with - | [] -> categories, args, Some (Parallelizable.Yes ClassParallelScope.Self) + | [] -> categories, args, mods, Some (Parallelizable.Yes ClassParallelScope.Self) | [ v ] -> match v.Value with | :? int as v -> match ParallelScope.ofInt v with | ParallelScope.Fixtures -> - categories, args, Some (Parallelizable.Yes ClassParallelScope.Fixtures) + categories, args, mods, Some (Parallelizable.Yes ClassParallelScope.Fixtures) | ParallelScope.Children -> - categories, args, Some (Parallelizable.Yes ClassParallelScope.Children) + categories, args, mods, Some (Parallelizable.Yes ClassParallelScope.Children) | ParallelScope.All -> - categories, args, Some (Parallelizable.Yes ClassParallelScope.All) + categories, args, mods, Some (Parallelizable.Yes ClassParallelScope.All) | ParallelScope.Self -> - categories, args, Some (Parallelizable.Yes ClassParallelScope.Self) - | ParallelScope.None -> categories, args, Some Parallelizable.No + categories, args, mods, Some (Parallelizable.Yes ClassParallelScope.Self) + | ParallelScope.None -> categories, args, mods, Some Parallelizable.No | v -> failwith $"Unexpectedly non-int value %O{v} of parallel scope in %s{parentType.FullName}" | _ -> failwith $"unexpectedly got multiple args to Parallelizable on %s{parentType.FullName}" - | _ -> categories, args, par + | "NUnit.Framework.ExplicitAttribute" -> + let reason = + attr.ConstructorArguments + |> Seq.tryHead + |> Option.map (_.Value >> unbox) + + categories, args, Modifier.Explicit reason :: mods, par + | "NUnit.Framework.IgnoreAttribute" -> + let reason = + attr.ConstructorArguments + |> Seq.tryHead + |> Option.map (_.Value >> unbox) + + categories, args, Modifier.Ignored reason :: mods, par + | _ -> categories, args, mods, par ) - (TestFixture.Empty parentType par args, parentType.GetRuntimeMethods ()) + (TestFixture.Empty parentType par mods args, parentType.GetRuntimeMethods ()) ||> Seq.fold (fun state mi -> ((state, []), mi.CustomAttributes) ||> Seq.fold (fun (state, unrecognisedAttrs) attr -> @@ -717,6 +731,8 @@ module TestFixture = /// Run every test (except those which fail the `filter`) in this test fixture, as well as the /// appropriate setup and tear-down logic. + /// + /// If the TestFixture has modifiers that specify no tests should be run, we don't run any tests. let run (contexts : TestContexts) (par : ParallelQueue) @@ -725,6 +741,26 @@ module TestFixture = (tests : TestFixture) : FixtureRunResults list Task = + match + tests.Modifiers + |> List.tryFind ( + function + | Modifier.Explicit _ + | Modifier.Ignored _ -> true + ) + with + | Some modifier -> + let reason = + match modifier with + | Modifier.Explicit (Some reason) -> reason + | Modifier.Ignored (Some reason) -> reason + | Modifier.Ignored None -> "test fixture marked Ignore" + | Modifier.Explicit None -> "test fixture marked Explicit" + + progress.OnTestFixtureSkipped tests.Name reason + Task.FromResult [] + | None -> + match tests.Parameters with | [] -> [ null ] | args -> args |> List.map List.toArray diff --git a/WoofWare.NUnitTestRunner.Lib/TestProgress.fs b/WoofWare.NUnitTestRunner.Lib/TestProgress.fs index 84fe86e..c5d9f3b 100644 --- a/WoofWare.NUnitTestRunner.Lib/TestProgress.fs +++ b/WoofWare.NUnitTestRunner.Lib/TestProgress.fs @@ -10,6 +10,8 @@ type ITestProgress = /// Called just before we start executing the setup logic for the given test fixture. /// We tell you how many test methods there are in the fixture. abstract OnTestFixtureStart : name : string -> testCount : int -> unit + /// Called when skipping the test fixture with the given name, e.g. because it's `[]`. + abstract OnTestFixtureSkipped : name : string -> reason : string -> unit /// Called just before we start executing the test(s) indicated by a particular method. abstract OnTestMemberStart : name : string -> unit /// Called when a test fails. (This may be called repeatedly with the same `name`, e.g. if the test @@ -32,6 +34,9 @@ module TestProgress = let plural = if testCount = 1 then "" else "s" writer.WriteLine $"Running test fixture: %s{name} (%i{testCount} test%s{plural} to run)" + member _.OnTestFixtureSkipped name reason = + writer.WriteLine $"Skipping test fixture (%s{reason}): %s{name}" + member _.OnTestMemberStart name = writer.WriteLine $"Running test: %s{name}" diff --git a/WoofWare.NUnitTestRunner.Lib/version.json b/WoofWare.NUnitTestRunner.Lib/version.json index 6c6d496..d1d2015 100644 --- a/WoofWare.NUnitTestRunner.Lib/version.json +++ b/WoofWare.NUnitTestRunner.Lib/version.json @@ -1,5 +1,5 @@ { - "version": "0.20", + "version": "0.21", "publicReleaseRefSpec": [ "^refs/heads/main$" ], diff --git a/WoofWare.NUnitTestRunner/Progress.fs b/WoofWare.NUnitTestRunner/Progress.fs index 9f026cc..6d45ef4 100644 --- a/WoofWare.NUnitTestRunner/Progress.fs +++ b/WoofWare.NUnitTestRunner/Progress.fs @@ -13,6 +13,10 @@ module Progress = member _.OnTestFixtureStart name testCount = console.MarkupLine $"[white]Running tests: %s{Markup.Escape name}[/]" + member _.OnTestFixtureSkipped name reason = + console.MarkupLine + $"[yellow]Skipping test fixture (%s{Markup.Escape reason}): %s{Markup.Escape name}[/]" + member _.OnTestMemberFinished name = console.MarkupLine $"[gray]Finished test: %s{Markup.Escape name}[/]"