From d1fa66a2e8dec328d642bb5119f9e8df8c93eca7 Mon Sep 17 00:00:00 2001
From: Patrick Stevens <3138005+Smaug123@users.noreply.github.com>
Date: Tue, 25 Mar 2025 21:49:50 +0000
Subject: [PATCH] Fix behaviour of `Explicit` test fixtures (#244)
---
Consumer/Consumer.fsproj | 1 +
Consumer/TestExplicit.fs | 24 +++++++
WoofWare.NUnitTestRunner.Lib/Domain.fs | 10 ++-
.../SurfaceBaseline.txt | 9 ++-
WoofWare.NUnitTestRunner.Lib/TestFixture.fs | 64 +++++++++++++++----
WoofWare.NUnitTestRunner.Lib/TestProgress.fs | 5 ++
WoofWare.NUnitTestRunner.Lib/version.json | 2 +-
WoofWare.NUnitTestRunner/Progress.fs | 4 ++
8 files changed, 100 insertions(+), 19 deletions(-)
create mode 100644 Consumer/TestExplicit.fs
diff --git a/Consumer/Consumer.fsproj b/Consumer/Consumer.fsproj
index 81e1242..519521e 100644
--- a/Consumer/Consumer.fsproj
+++ b/Consumer/Consumer.fsproj
@@ -10,6 +10,7 @@
+
diff --git a/Consumer/TestExplicit.fs b/Consumer/TestExplicit.fs
new file mode 100644
index 0000000..c1f24c6
--- /dev/null
+++ b/Consumer/TestExplicit.fs
@@ -0,0 +1,24 @@
+namespace Consumer
+
+open NUnit.Framework
+
+[]
+module TestExplicitIndividual =
+
+ []
+ []
+ let ``This test should not be run`` () = failwith "should not call"
+
+[]
+[]
+module TestExplicitModule =
+
+ []
+ let setUp () = failwith "should not call: setup"
+
+ []
+ let tearDown () =
+ failwith "should not call: teardown"
+
+ []
+ let ``This test should not be run because its module is explicit`` () = failwith "should not call: test"
diff --git a/WoofWare.NUnitTestRunner.Lib/Domain.fs b/WoofWare.NUnitTestRunner.Lib/Domain.fs
index 22d1b51..10200e4 100644
--- a/WoofWare.NUnitTestRunner.Lib/Domain.fs
+++ b/WoofWare.NUnitTestRunner.Lib/Domain.fs
@@ -138,10 +138,17 @@ type TestFixture =
Tests : SingleTestMethod list
/// If this fixture has declared a parallelisability, that goes here.
Parallelize : Parallelizable option
+ /// It is possible to mark a fixture as "Explicit" or "Ignored", for example.
+ Modifiers : Modifier list
}
/// A test fixture about which we know nothing. No tests, no setup/teardown.
- static member Empty (ty : Type) (par : Parallelizable option) (args : obj list list) =
+ static member Empty
+ (ty : Type)
+ (par : Parallelizable option)
+ (modifiers : Modifier list)
+ (args : obj list list)
+ =
{
ContainingAssembly = ty.Assembly
Type = ty
@@ -153,6 +160,7 @@ type TestFixture =
Parameters = args
Tests = []
Parallelize = par
+ Modifiers = modifiers
}
/// User code in the unit under test has failed somehow.
diff --git a/WoofWare.NUnitTestRunner.Lib/SurfaceBaseline.txt b/WoofWare.NUnitTestRunner.Lib/SurfaceBaseline.txt
index 1ba69f1..2791eb8 100644
--- a/WoofWare.NUnitTestRunner.Lib/SurfaceBaseline.txt
+++ b/WoofWare.NUnitTestRunner.Lib/SurfaceBaseline.txt
@@ -170,8 +170,9 @@ WoofWare.NUnitTestRunner.IndividualTestRunMetadata.StdOut [property]: [read-only
WoofWare.NUnitTestRunner.IndividualTestRunMetadata.TestId [property]: [read-only] System.Guid
WoofWare.NUnitTestRunner.IndividualTestRunMetadata.TestName [property]: [read-only] string
WoofWare.NUnitTestRunner.IndividualTestRunMetadata.Total [property]: [read-only] System.TimeSpan
-WoofWare.NUnitTestRunner.ITestProgress - interface with 5 member(s)
+WoofWare.NUnitTestRunner.ITestProgress - interface with 6 member(s)
WoofWare.NUnitTestRunner.ITestProgress.OnTestFailed [method]: string -> WoofWare.NUnitTestRunner.TestMemberFailure -> unit
+WoofWare.NUnitTestRunner.ITestProgress.OnTestFixtureSkipped [method]: string -> string -> unit
WoofWare.NUnitTestRunner.ITestProgress.OnTestFixtureStart [method]: string -> int -> unit
WoofWare.NUnitTestRunner.ITestProgress.OnTestMemberFinished [method]: string -> unit
WoofWare.NUnitTestRunner.ITestProgress.OnTestMemberSkipped [method]: string -> unit
@@ -346,11 +347,12 @@ WoofWare.NUnitTestRunner.TestFailure.NewTearDownFailed [static method]: WoofWare
WoofWare.NUnitTestRunner.TestFailure.NewTestFailed [static method]: WoofWare.NUnitTestRunner.UserMethodFailure -> WoofWare.NUnitTestRunner.TestFailure
WoofWare.NUnitTestRunner.TestFailure.Tag [property]: [read-only] int
WoofWare.NUnitTestRunner.TestFixture inherit obj, implements WoofWare.NUnitTestRunner.TestFixture System.IEquatable, System.Collections.IStructuralEquatable
-WoofWare.NUnitTestRunner.TestFixture..ctor [constructor]: (System.Reflection.Assembly, string, System.Type, System.Reflection.MethodInfo option, System.Reflection.MethodInfo option, System.Reflection.MethodInfo list, System.Reflection.MethodInfo list, obj list list, WoofWare.NUnitTestRunner.SingleTestMethod list, WoofWare.NUnitTestRunner.ClassParallelScope WoofWare.NUnitTestRunner.Parallelizable option)
+WoofWare.NUnitTestRunner.TestFixture..ctor [constructor]: (System.Reflection.Assembly, string, System.Type, System.Reflection.MethodInfo option, System.Reflection.MethodInfo option, System.Reflection.MethodInfo list, System.Reflection.MethodInfo list, obj list list, WoofWare.NUnitTestRunner.SingleTestMethod list, WoofWare.NUnitTestRunner.ClassParallelScope WoofWare.NUnitTestRunner.Parallelizable option, WoofWare.NUnitTestRunner.Modifier list)
WoofWare.NUnitTestRunner.TestFixture.ContainingAssembly [property]: [read-only] System.Reflection.Assembly
-WoofWare.NUnitTestRunner.TestFixture.Empty [static method]: System.Type -> WoofWare.NUnitTestRunner.ClassParallelScope WoofWare.NUnitTestRunner.Parallelizable option -> obj list list -> WoofWare.NUnitTestRunner.TestFixture
+WoofWare.NUnitTestRunner.TestFixture.Empty [static method]: System.Type -> WoofWare.NUnitTestRunner.ClassParallelScope WoofWare.NUnitTestRunner.Parallelizable option -> WoofWare.NUnitTestRunner.Modifier list -> obj list list -> WoofWare.NUnitTestRunner.TestFixture
WoofWare.NUnitTestRunner.TestFixture.Equals [method]: (WoofWare.NUnitTestRunner.TestFixture, System.Collections.IEqualityComparer) -> bool
WoofWare.NUnitTestRunner.TestFixture.get_ContainingAssembly [method]: unit -> System.Reflection.Assembly
+WoofWare.NUnitTestRunner.TestFixture.get_Modifiers [method]: unit -> WoofWare.NUnitTestRunner.Modifier list
WoofWare.NUnitTestRunner.TestFixture.get_Name [method]: unit -> string
WoofWare.NUnitTestRunner.TestFixture.get_OneTimeSetUp [method]: unit -> System.Reflection.MethodInfo option
WoofWare.NUnitTestRunner.TestFixture.get_OneTimeTearDown [method]: unit -> System.Reflection.MethodInfo option
@@ -360,6 +362,7 @@ WoofWare.NUnitTestRunner.TestFixture.get_SetUp [method]: unit -> System.Reflecti
WoofWare.NUnitTestRunner.TestFixture.get_TearDown [method]: unit -> System.Reflection.MethodInfo list
WoofWare.NUnitTestRunner.TestFixture.get_Tests [method]: unit -> WoofWare.NUnitTestRunner.SingleTestMethod list
WoofWare.NUnitTestRunner.TestFixture.get_Type [method]: unit -> System.Type
+WoofWare.NUnitTestRunner.TestFixture.Modifiers [property]: [read-only] WoofWare.NUnitTestRunner.Modifier list
WoofWare.NUnitTestRunner.TestFixture.Name [property]: [read-only] string
WoofWare.NUnitTestRunner.TestFixture.OneTimeSetUp [property]: [read-only] System.Reflection.MethodInfo option
WoofWare.NUnitTestRunner.TestFixture.OneTimeTearDown [property]: [read-only] System.Reflection.MethodInfo option
diff --git a/WoofWare.NUnitTestRunner.Lib/TestFixture.fs b/WoofWare.NUnitTestRunner.Lib/TestFixture.fs
index 9dc13dd..3260b39 100644
--- a/WoofWare.NUnitTestRunner.Lib/TestFixture.fs
+++ b/WoofWare.NUnitTestRunner.Lib/TestFixture.fs
@@ -597,15 +597,15 @@ module TestFixture =
/// Interpret this type as a [], extracting the test members from it and annotating them with all
/// relevant information about how we should run them.
let parse (parentType : Type) : TestFixture =
- let categories, args, par =
- (([], [], None), parentType.CustomAttributes)
- ||> Seq.fold (fun (categories, args, par) attr ->
+ let categories, args, mods, par =
+ (([], [], [], None), parentType.CustomAttributes)
+ ||> Seq.fold (fun (categories, args, mods, par) attr ->
match attr.AttributeType.FullName with
| "NUnit.Framework.SetUpFixtureAttribute" ->
failwith "This test runner does not support SetUpFixture. Please shout if you want this."
| "NUnit.Framework.CategoryAttribute" ->
let cat = attr.ConstructorArguments |> Seq.exactlyOne |> _.Value |> unbox
- cat :: categories, args, par
+ cat :: categories, args, mods, par
| "NUnit.Framework.TestFixtureAttribute" ->
let newArgs =
match attr.ConstructorArguments |> Seq.map _.Value |> Seq.toList with
@@ -613,38 +613,52 @@ module TestFixture =
x |> Seq.cast |> Seq.map _.Value |> Seq.toList
| xs -> xs
- categories, newArgs :: args, par
+ categories, newArgs :: args, mods, par
| "NUnit.Framework.NonParallelizableAttribute" ->
match par with
| Some _ -> failwith $"Got multiple parallelism attributes on %s{parentType.FullName}"
- | None -> categories, args, Some Parallelizable.No
+ | None -> categories, args, mods, Some Parallelizable.No
| "NUnit.Framework.ParallelizableAttribute" ->
match par with
| Some _ -> failwith $"Got multiple parallelism attributes on %s{parentType.FullName}"
| None ->
match attr.ConstructorArguments |> Seq.toList with
- | [] -> categories, args, Some (Parallelizable.Yes ClassParallelScope.Self)
+ | [] -> categories, args, mods, Some (Parallelizable.Yes ClassParallelScope.Self)
| [ v ] ->
match v.Value with
| :? int as v ->
match ParallelScope.ofInt v with
| ParallelScope.Fixtures ->
- categories, args, Some (Parallelizable.Yes ClassParallelScope.Fixtures)
+ categories, args, mods, Some (Parallelizable.Yes ClassParallelScope.Fixtures)
| ParallelScope.Children ->
- categories, args, Some (Parallelizable.Yes ClassParallelScope.Children)
+ categories, args, mods, Some (Parallelizable.Yes ClassParallelScope.Children)
| ParallelScope.All ->
- categories, args, Some (Parallelizable.Yes ClassParallelScope.All)
+ categories, args, mods, Some (Parallelizable.Yes ClassParallelScope.All)
| ParallelScope.Self ->
- categories, args, Some (Parallelizable.Yes ClassParallelScope.Self)
- | ParallelScope.None -> categories, args, Some Parallelizable.No
+ categories, args, mods, Some (Parallelizable.Yes ClassParallelScope.Self)
+ | ParallelScope.None -> categories, args, mods, Some Parallelizable.No
| v ->
failwith
$"Unexpectedly non-int value %O{v} of parallel scope in %s{parentType.FullName}"
| _ -> failwith $"unexpectedly got multiple args to Parallelizable on %s{parentType.FullName}"
- | _ -> categories, args, par
+ | "NUnit.Framework.ExplicitAttribute" ->
+ let reason =
+ attr.ConstructorArguments
+ |> Seq.tryHead
+ |> Option.map (_.Value >> unbox)
+
+ categories, args, Modifier.Explicit reason :: mods, par
+ | "NUnit.Framework.IgnoreAttribute" ->
+ let reason =
+ attr.ConstructorArguments
+ |> Seq.tryHead
+ |> Option.map (_.Value >> unbox)
+
+ categories, args, Modifier.Ignored reason :: mods, par
+ | _ -> categories, args, mods, par
)
- (TestFixture.Empty parentType par args, parentType.GetRuntimeMethods ())
+ (TestFixture.Empty parentType par mods args, parentType.GetRuntimeMethods ())
||> Seq.fold (fun state mi ->
((state, []), mi.CustomAttributes)
||> Seq.fold (fun (state, unrecognisedAttrs) attr ->
@@ -717,6 +731,8 @@ module TestFixture =
/// Run every test (except those which fail the `filter`) in this test fixture, as well as the
/// appropriate setup and tear-down logic.
+ ///
+ /// If the TestFixture has modifiers that specify no tests should be run, we don't run any tests.
let run
(contexts : TestContexts)
(par : ParallelQueue)
@@ -725,6 +741,26 @@ module TestFixture =
(tests : TestFixture)
: FixtureRunResults list Task
=
+ match
+ tests.Modifiers
+ |> List.tryFind (
+ function
+ | Modifier.Explicit _
+ | Modifier.Ignored _ -> true
+ )
+ with
+ | Some modifier ->
+ let reason =
+ match modifier with
+ | Modifier.Explicit (Some reason) -> reason
+ | Modifier.Ignored (Some reason) -> reason
+ | Modifier.Ignored None -> "test fixture marked Ignore"
+ | Modifier.Explicit None -> "test fixture marked Explicit"
+
+ progress.OnTestFixtureSkipped tests.Name reason
+ Task.FromResult []
+ | None ->
+
match tests.Parameters with
| [] -> [ null ]
| args -> args |> List.map List.toArray
diff --git a/WoofWare.NUnitTestRunner.Lib/TestProgress.fs b/WoofWare.NUnitTestRunner.Lib/TestProgress.fs
index 84fe86e..c5d9f3b 100644
--- a/WoofWare.NUnitTestRunner.Lib/TestProgress.fs
+++ b/WoofWare.NUnitTestRunner.Lib/TestProgress.fs
@@ -10,6 +10,8 @@ type ITestProgress =
/// Called just before we start executing the setup logic for the given test fixture.
/// We tell you how many test methods there are in the fixture.
abstract OnTestFixtureStart : name : string -> testCount : int -> unit
+ /// Called when skipping the test fixture with the given name, e.g. because it's `[]`.
+ abstract OnTestFixtureSkipped : name : string -> reason : string -> unit
/// Called just before we start executing the test(s) indicated by a particular method.
abstract OnTestMemberStart : name : string -> unit
/// Called when a test fails. (This may be called repeatedly with the same `name`, e.g. if the test
@@ -32,6 +34,9 @@ module TestProgress =
let plural = if testCount = 1 then "" else "s"
writer.WriteLine $"Running test fixture: %s{name} (%i{testCount} test%s{plural} to run)"
+ member _.OnTestFixtureSkipped name reason =
+ writer.WriteLine $"Skipping test fixture (%s{reason}): %s{name}"
+
member _.OnTestMemberStart name =
writer.WriteLine $"Running test: %s{name}"
diff --git a/WoofWare.NUnitTestRunner.Lib/version.json b/WoofWare.NUnitTestRunner.Lib/version.json
index 6c6d496..d1d2015 100644
--- a/WoofWare.NUnitTestRunner.Lib/version.json
+++ b/WoofWare.NUnitTestRunner.Lib/version.json
@@ -1,5 +1,5 @@
{
- "version": "0.20",
+ "version": "0.21",
"publicReleaseRefSpec": [
"^refs/heads/main$"
],
diff --git a/WoofWare.NUnitTestRunner/Progress.fs b/WoofWare.NUnitTestRunner/Progress.fs
index 9f026cc..6d45ef4 100644
--- a/WoofWare.NUnitTestRunner/Progress.fs
+++ b/WoofWare.NUnitTestRunner/Progress.fs
@@ -13,6 +13,10 @@ module Progress =
member _.OnTestFixtureStart name testCount =
console.MarkupLine $"[white]Running tests: %s{Markup.Escape name}[/]"
+ member _.OnTestFixtureSkipped name reason =
+ console.MarkupLine
+ $"[yellow]Skipping test fixture (%s{Markup.Escape reason}): %s{Markup.Escape name}[/]"
+
member _.OnTestMemberFinished name =
console.MarkupLine $"[gray]Finished test: %s{Markup.Escape name}[/]"