Fix behaviour of Explicit test fixtures (#244)

Consumer/Consumer.fsproj

@@ -10,6 +10,7 @@
         <Compile Include="NoAttribute.fs" />
         <Compile Include="Inconclusive.fs" />
         <Compile Include="RunSubProcess.fs" />
+        <Compile Include="TestExplicit.fs" />
         <Compile Include="TestNonParallel.fs" />
         <Compile Include="TestParallel.fs" />
         <Compile Include="TestParallelIndividualTest.fs" />

Consumer/TestExplicit.fs

@@ -0,0 +1,24 @@
+namespace Consumer
+
+open NUnit.Framework
+
+[<TestFixture>]
+module TestExplicitIndividual =
+
+    [<Explicit>]
+    [<Test>]
+    let ``This test should not be run`` () = failwith<unit> "should not call"
+
+[<Explicit>]
+[<TestFixture>]
+module TestExplicitModule =
+
+    [<OneTimeSetUp>]
+    let setUp () = failwith<unit> "should not call: setup"
+
+    [<OneTimeTearDown>]
+    let tearDown () =
+        failwith<unit> "should not call: teardown"
+
+    [<Test>]
+    let ``This test should not be run because its module is explicit`` () = failwith<unit> "should not call: test"

WoofWare.NUnitTestRunner.Lib/Domain.fs

@@ -138,10 +138,17 @@ type TestFixture =
         Tests : SingleTestMethod list
         /// If this fixture has declared a parallelisability, that goes here.
         Parallelize : Parallelizable<ClassParallelScope> option
+        /// It is possible to mark a fixture as "Explicit" or "Ignored", for example.
+        Modifiers : Modifier list
     }
 
     /// A test fixture about which we know nothing. No tests, no setup/teardown.
-    static member Empty (ty : Type) (par : Parallelizable<ClassParallelScope> option) (args : obj list list) =
+    static member Empty
+        (ty : Type)
+        (par : Parallelizable<ClassParallelScope> option)
+        (modifiers : Modifier list)
+        (args : obj list list)
+        =
         {
             ContainingAssembly = ty.Assembly
             Type = ty
@@ -153,6 +160,7 @@ type TestFixture =
             Parameters = args
             Tests = []
             Parallelize = par
+            Modifiers = modifiers
         }
 
 /// User code in the unit under test has failed somehow.

WoofWare.NUnitTestRunner.Lib/SurfaceBaseline.txt

@@ -170,8 +170,9 @@ WoofWare.NUnitTestRunner.IndividualTestRunMetadata.StdOut [property]: [read-only
 WoofWare.NUnitTestRunner.IndividualTestRunMetadata.TestId [property]: [read-only] System.Guid
 WoofWare.NUnitTestRunner.IndividualTestRunMetadata.TestName [property]: [read-only] string
 WoofWare.NUnitTestRunner.IndividualTestRunMetadata.Total [property]: [read-only] System.TimeSpan
-WoofWare.NUnitTestRunner.ITestProgress - interface with 5 member(s)
+WoofWare.NUnitTestRunner.ITestProgress - interface with 6 member(s)
 WoofWare.NUnitTestRunner.ITestProgress.OnTestFailed [method]: string -> WoofWare.NUnitTestRunner.TestMemberFailure -> unit
+WoofWare.NUnitTestRunner.ITestProgress.OnTestFixtureSkipped [method]: string -> string -> unit
 WoofWare.NUnitTestRunner.ITestProgress.OnTestFixtureStart [method]: string -> int -> unit
 WoofWare.NUnitTestRunner.ITestProgress.OnTestMemberFinished [method]: string -> unit
 WoofWare.NUnitTestRunner.ITestProgress.OnTestMemberSkipped [method]: string -> unit
@@ -346,11 +347,12 @@ WoofWare.NUnitTestRunner.TestFailure.NewTearDownFailed [static method]: WoofWare
 WoofWare.NUnitTestRunner.TestFailure.NewTestFailed [static method]: WoofWare.NUnitTestRunner.UserMethodFailure -> WoofWare.NUnitTestRunner.TestFailure
 WoofWare.NUnitTestRunner.TestFailure.Tag [property]: [read-only] int
 WoofWare.NUnitTestRunner.TestFixture inherit obj, implements WoofWare.NUnitTestRunner.TestFixture System.IEquatable, System.Collections.IStructuralEquatable
-WoofWare.NUnitTestRunner.TestFixture..ctor [constructor]: (System.Reflection.Assembly, string, System.Type, System.Reflection.MethodInfo option, System.Reflection.MethodInfo option, System.Reflection.MethodInfo list, System.Reflection.MethodInfo list, obj list list, WoofWare.NUnitTestRunner.SingleTestMethod list, WoofWare.NUnitTestRunner.ClassParallelScope WoofWare.NUnitTestRunner.Parallelizable option)
+WoofWare.NUnitTestRunner.TestFixture..ctor [constructor]: (System.Reflection.Assembly, string, System.Type, System.Reflection.MethodInfo option, System.Reflection.MethodInfo option, System.Reflection.MethodInfo list, System.Reflection.MethodInfo list, obj list list, WoofWare.NUnitTestRunner.SingleTestMethod list, WoofWare.NUnitTestRunner.ClassParallelScope WoofWare.NUnitTestRunner.Parallelizable option, WoofWare.NUnitTestRunner.Modifier list)
 WoofWare.NUnitTestRunner.TestFixture.ContainingAssembly [property]: [read-only] System.Reflection.Assembly
-WoofWare.NUnitTestRunner.TestFixture.Empty [static method]: System.Type -> WoofWare.NUnitTestRunner.ClassParallelScope WoofWare.NUnitTestRunner.Parallelizable option -> obj list list -> WoofWare.NUnitTestRunner.TestFixture
+WoofWare.NUnitTestRunner.TestFixture.Empty [static method]: System.Type -> WoofWare.NUnitTestRunner.ClassParallelScope WoofWare.NUnitTestRunner.Parallelizable option -> WoofWare.NUnitTestRunner.Modifier list -> obj list list -> WoofWare.NUnitTestRunner.TestFixture
 WoofWare.NUnitTestRunner.TestFixture.Equals [method]: (WoofWare.NUnitTestRunner.TestFixture, System.Collections.IEqualityComparer) -> bool
 WoofWare.NUnitTestRunner.TestFixture.get_ContainingAssembly [method]: unit -> System.Reflection.Assembly
+WoofWare.NUnitTestRunner.TestFixture.get_Modifiers [method]: unit -> WoofWare.NUnitTestRunner.Modifier list
 WoofWare.NUnitTestRunner.TestFixture.get_Name [method]: unit -> string
 WoofWare.NUnitTestRunner.TestFixture.get_OneTimeSetUp [method]: unit -> System.Reflection.MethodInfo option
 WoofWare.NUnitTestRunner.TestFixture.get_OneTimeTearDown [method]: unit -> System.Reflection.MethodInfo option
@@ -360,6 +362,7 @@ WoofWare.NUnitTestRunner.TestFixture.get_SetUp [method]: unit -> System.Reflecti
 WoofWare.NUnitTestRunner.TestFixture.get_TearDown [method]: unit -> System.Reflection.MethodInfo list
 WoofWare.NUnitTestRunner.TestFixture.get_Tests [method]: unit -> WoofWare.NUnitTestRunner.SingleTestMethod list
 WoofWare.NUnitTestRunner.TestFixture.get_Type [method]: unit -> System.Type
+WoofWare.NUnitTestRunner.TestFixture.Modifiers [property]: [read-only] WoofWare.NUnitTestRunner.Modifier list
 WoofWare.NUnitTestRunner.TestFixture.Name [property]: [read-only] string
 WoofWare.NUnitTestRunner.TestFixture.OneTimeSetUp [property]: [read-only] System.Reflection.MethodInfo option
 WoofWare.NUnitTestRunner.TestFixture.OneTimeTearDown [property]: [read-only] System.Reflection.MethodInfo option

WoofWare.NUnitTestRunner.Lib/TestFixture.fs

@@ -597,15 +597,15 @@ module TestFixture =
     /// Interpret this type as a [<TestFixture>], extracting the test members from it and annotating them with all
     /// relevant information about how we should run them.
     let parse (parentType : Type) : TestFixture =
-        let categories, args, par =
-            (([], [], None), parentType.CustomAttributes)
-            ||> Seq.fold (fun (categories, args, par) attr ->
+        let categories, args, mods, par =
+            (([], [], [], None), parentType.CustomAttributes)
+            ||> Seq.fold (fun (categories, args, mods, par) attr ->
                 match attr.AttributeType.FullName with
                 | "NUnit.Framework.SetUpFixtureAttribute" ->
                     failwith "This test runner does not support SetUpFixture. Please shout if you want this."
                 | "NUnit.Framework.CategoryAttribute" ->
                     let cat = attr.ConstructorArguments |> Seq.exactlyOne |> _.Value |> unbox<string>
-                    cat :: categories, args, par
+                    cat :: categories, args, mods, par
                 | "NUnit.Framework.TestFixtureAttribute" ->
                     let newArgs =
                         match attr.ConstructorArguments |> Seq.map _.Value |> Seq.toList with
@@ -613,38 +613,52 @@ module TestFixture =
                             x |> Seq.cast<CustomAttributeTypedArgument> |> Seq.map _.Value |> Seq.toList
                         | xs -> xs
 
-                    categories, newArgs :: args, par
+                    categories, newArgs :: args, mods, par
                 | "NUnit.Framework.NonParallelizableAttribute" ->
                     match par with
                     | Some _ -> failwith $"Got multiple parallelism attributes on %s{parentType.FullName}"
-                    | None -> categories, args, Some Parallelizable.No
+                    | None -> categories, args, mods, Some Parallelizable.No
                 | "NUnit.Framework.ParallelizableAttribute" ->
                     match par with
                     | Some _ -> failwith $"Got multiple parallelism attributes on %s{parentType.FullName}"
                     | None ->
                         match attr.ConstructorArguments |> Seq.toList with
-                        | [] -> categories, args, Some (Parallelizable.Yes ClassParallelScope.Self)
+                        | [] -> categories, args, mods, Some (Parallelizable.Yes ClassParallelScope.Self)
                         | [ v ] ->
                             match v.Value with
                             | :? int as v ->
                                 match ParallelScope.ofInt v with
                                 | ParallelScope.Fixtures ->
-                                    categories, args, Some (Parallelizable.Yes ClassParallelScope.Fixtures)
+                                    categories, args, mods, Some (Parallelizable.Yes ClassParallelScope.Fixtures)
                                 | ParallelScope.Children ->
-                                    categories, args, Some (Parallelizable.Yes ClassParallelScope.Children)
+                                    categories, args, mods, Some (Parallelizable.Yes ClassParallelScope.Children)
                                 | ParallelScope.All ->
-                                    categories, args, Some (Parallelizable.Yes ClassParallelScope.All)
+                                    categories, args, mods, Some (Parallelizable.Yes ClassParallelScope.All)
                                 | ParallelScope.Self ->
-                                    categories, args, Some (Parallelizable.Yes ClassParallelScope.Self)
-                                | ParallelScope.None -> categories, args, Some Parallelizable.No
+                                    categories, args, mods, Some (Parallelizable.Yes ClassParallelScope.Self)
+                                | ParallelScope.None -> categories, args, mods, Some Parallelizable.No
                             | v ->
                                 failwith
                                     $"Unexpectedly non-int value %O{v} of parallel scope in %s{parentType.FullName}"
                         | _ -> failwith $"unexpectedly got multiple args to Parallelizable on %s{parentType.FullName}"
-                | _ -> categories, args, par
+                | "NUnit.Framework.ExplicitAttribute" ->
+                    let reason =
+                        attr.ConstructorArguments
+                        |> Seq.tryHead
+                        |> Option.map (_.Value >> unbox<string>)
+
+                    categories, args, Modifier.Explicit reason :: mods, par
+                | "NUnit.Framework.IgnoreAttribute" ->
+                    let reason =
+                        attr.ConstructorArguments
+                        |> Seq.tryHead
+                        |> Option.map (_.Value >> unbox<string>)
+
+                    categories, args, Modifier.Ignored reason :: mods, par
+                | _ -> categories, args, mods, par
             )
 
-        (TestFixture.Empty parentType par args, parentType.GetRuntimeMethods ())
+        (TestFixture.Empty parentType par mods args, parentType.GetRuntimeMethods ())
         ||> Seq.fold (fun state mi ->
             ((state, []), mi.CustomAttributes)
             ||> Seq.fold (fun (state, unrecognisedAttrs) attr ->
@@ -717,6 +731,8 @@ module TestFixture =
 
     /// Run every test (except those which fail the `filter`) in this test fixture, as well as the
     /// appropriate setup and tear-down logic.
+    ///
+    /// If the TestFixture has modifiers that specify no tests should be run, we don't run any tests.
     let run
         (contexts : TestContexts)
         (par : ParallelQueue)
@@ -725,6 +741,26 @@ module TestFixture =
         (tests : TestFixture)
         : FixtureRunResults list Task
         =
+        match
+            tests.Modifiers
+            |> List.tryFind (
+                function
+                | Modifier.Explicit _
+                | Modifier.Ignored _ -> true
+            )
+        with
+        | Some modifier ->
+            let reason =
+                match modifier with
+                | Modifier.Explicit (Some reason) -> reason
+                | Modifier.Ignored (Some reason) -> reason
+                | Modifier.Ignored None -> "test fixture marked Ignore"
+                | Modifier.Explicit None -> "test fixture marked Explicit"
+
+            progress.OnTestFixtureSkipped tests.Name reason
+            Task.FromResult []
+        | None ->
+
         match tests.Parameters with
         | [] -> [ null ]
         | args -> args |> List.map List.toArray

WoofWare.NUnitTestRunner.Lib/TestProgress.fs

@@ -10,6 +10,8 @@ type ITestProgress =
     /// Called just before we start executing the setup logic for the given test fixture.
     /// We tell you how many test methods there are in the fixture.
     abstract OnTestFixtureStart : name : string -> testCount : int -> unit
+    /// Called when skipping the test fixture with the given name, e.g. because it's `[<Explicit>]`.
+    abstract OnTestFixtureSkipped : name : string -> reason : string -> unit
     /// Called just before we start executing the test(s) indicated by a particular method.
     abstract OnTestMemberStart : name : string -> unit
     /// Called when a test fails. (This may be called repeatedly with the same `name`, e.g. if the test
@@ -32,6 +34,9 @@ module TestProgress =
                 let plural = if testCount = 1 then "" else "s"
                 writer.WriteLine $"Running test fixture: %s{name} (%i{testCount} test%s{plural} to run)"
 
+            member _.OnTestFixtureSkipped name reason =
+                writer.WriteLine $"Skipping test fixture (%s{reason}): %s{name}"
+
             member _.OnTestMemberStart name =
                 writer.WriteLine $"Running test: %s{name}"
 

WoofWare.NUnitTestRunner.Lib/version.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.20",
+  "version": "0.21",
   "publicReleaseRefSpec": [
     "^refs/heads/main$"
   ],

WoofWare.NUnitTestRunner/Progress.fs

@@ -13,6 +13,10 @@ module Progress =
             member _.OnTestFixtureStart name testCount =
                 console.MarkupLine $"[white]Running tests: %s{Markup.Escape name}[/]"
 
+            member _.OnTestFixtureSkipped name reason =
+                console.MarkupLine
+                    $"[yellow]Skipping test fixture (%s{Markup.Escape reason}): %s{Markup.Escape name}[/]"
+
             member _.OnTestMemberFinished name =
                 console.MarkupLine $"[gray]Finished test: %s{Markup.Escape name}[/]"