Cap runtime of test (#73)

Consumer/Consumer.fsproj

@@ -10,6 +10,7 @@
     <ItemGroup>
         <Compile Include="NoAttribute.fs" />
         <Compile Include="Inconclusive.fs" />
+        <Compile Include="TestParameterisedFixture.fs" />
         <Compile Include="TestSetUp.fs" />
         <Compile Include="TestValues.fs" />
         <None Include="some-config.json">

Consumer/TestParameterisedFixture.fs

@@ -0,0 +1,18 @@
+namespace Consumer
+
+open NUnit.Framework
+open FsUnitTyped
+
+[<TestFixture true>]
+[<TestFixture false>]
+type TestParameterisedFixture (v : bool) =
+    [<Test>]
+    member _.Thing () = v |> shouldEqual v
+
+[<TestFixture(3, true)>]
+[<TestFixture(6, false)>]
+type TestParameterisedFixtureMultiple (i : int, v : bool) =
+    [<Test>]
+    member _.Thing () =
+        v |> shouldEqual v
+        i |> shouldEqual i

WoofWare.NUnitTestRunner.Lib/Domain.fs

@@ -1,5 +1,6 @@
 namespace WoofWare.NUnitTestRunner
 
+open System
 open System.Reflection
 
 /// A modifier on whether a given test should be run.
@@ -62,6 +63,8 @@ type TestFixture =
         /// Fully-qualified name of this fixture (e.g. MyThing.Test.Foo for `[<TestFixture>] module Foo` in the
         /// `MyThing.Test` assembly).
         Name : string
+        /// The type which is this fixture, containing the tests as members.
+        Type : Type
         /// A method which is run once when this test fixture starts, before any other setup logic and before
         /// any tests run. If this method fails, no tests will run and no per-test setup/teardown logic will run,
         /// but OneTimeTearDown will run.
@@ -77,19 +80,24 @@ type TestFixture =
         /// Methods which are run in some arbitrary order after each individual test, even if the test or its setup
         /// failed. If the first TearDown we run fails, we don't define whether the other TearDowns run.
         TearDown : MethodInfo list
+        /// You might have defined e.g. `[<TestFixture true>] type Foo (v : bool) = ...`. If so, this gives the
+        /// various possible parameters.
+        Parameters : obj list list
         /// The individual test methods present within this fixture.
         Tests : SingleTestMethod list
     }
 
     /// A test fixture about which we know nothing. No tests, no setup/teardown.
-    static member Empty (containingAssembly : Assembly) (name : string) =
+    static member Empty (ty : Type) (args : obj list list) =
         {
-            ContainingAssembly = containingAssembly
+            ContainingAssembly = ty.Assembly
-            Name = name
+            Type = ty
+            Name = ty.Name
             OneTimeSetUp = None
             OneTimeTearDown = None
             SetUp = []
             TearDown = []
+            Parameters = args
             Tests = []
         }
 

WoofWare.NUnitTestRunner.Lib/SurfaceBaseline.txt

@@ -185,25 +185,29 @@ WoofWare.NUnitTestRunner.TestFailure.NewTearDownFailed [static method]: WoofWare
 WoofWare.NUnitTestRunner.TestFailure.NewTestFailed [static method]: WoofWare.NUnitTestRunner.UserMethodFailure -> WoofWare.NUnitTestRunner.TestFailure
 WoofWare.NUnitTestRunner.TestFailure.Tag [property]: [read-only] int
 WoofWare.NUnitTestRunner.TestFixture inherit obj, implements WoofWare.NUnitTestRunner.TestFixture System.IEquatable, System.Collections.IStructuralEquatable
-WoofWare.NUnitTestRunner.TestFixture..ctor [constructor]: (System.Reflection.Assembly, string, System.Reflection.MethodInfo option, System.Reflection.MethodInfo option, System.Reflection.MethodInfo list, System.Reflection.MethodInfo list, WoofWare.NUnitTestRunner.SingleTestMethod list)
+WoofWare.NUnitTestRunner.TestFixture..ctor [constructor]: (System.Reflection.Assembly, string, System.Type, System.Reflection.MethodInfo option, System.Reflection.MethodInfo option, System.Reflection.MethodInfo list, System.Reflection.MethodInfo list, obj list list, WoofWare.NUnitTestRunner.SingleTestMethod list)
 WoofWare.NUnitTestRunner.TestFixture.ContainingAssembly [property]: [read-only] System.Reflection.Assembly
-WoofWare.NUnitTestRunner.TestFixture.Empty [static method]: System.Reflection.Assembly -> string -> WoofWare.NUnitTestRunner.TestFixture
+WoofWare.NUnitTestRunner.TestFixture.Empty [static method]: System.Type -> obj list list -> WoofWare.NUnitTestRunner.TestFixture
 WoofWare.NUnitTestRunner.TestFixture.get_ContainingAssembly [method]: unit -> System.Reflection.Assembly
 WoofWare.NUnitTestRunner.TestFixture.get_Name [method]: unit -> string
 WoofWare.NUnitTestRunner.TestFixture.get_OneTimeSetUp [method]: unit -> System.Reflection.MethodInfo option
 WoofWare.NUnitTestRunner.TestFixture.get_OneTimeTearDown [method]: unit -> System.Reflection.MethodInfo option
+WoofWare.NUnitTestRunner.TestFixture.get_Parameters [method]: unit -> obj list list
 WoofWare.NUnitTestRunner.TestFixture.get_SetUp [method]: unit -> System.Reflection.MethodInfo list
 WoofWare.NUnitTestRunner.TestFixture.get_TearDown [method]: unit -> System.Reflection.MethodInfo list
 WoofWare.NUnitTestRunner.TestFixture.get_Tests [method]: unit -> WoofWare.NUnitTestRunner.SingleTestMethod list
+WoofWare.NUnitTestRunner.TestFixture.get_Type [method]: unit -> System.Type
 WoofWare.NUnitTestRunner.TestFixture.Name [property]: [read-only] string
 WoofWare.NUnitTestRunner.TestFixture.OneTimeSetUp [property]: [read-only] System.Reflection.MethodInfo option
 WoofWare.NUnitTestRunner.TestFixture.OneTimeTearDown [property]: [read-only] System.Reflection.MethodInfo option
+WoofWare.NUnitTestRunner.TestFixture.Parameters [property]: [read-only] obj list list
 WoofWare.NUnitTestRunner.TestFixture.SetUp [property]: [read-only] System.Reflection.MethodInfo list
 WoofWare.NUnitTestRunner.TestFixture.TearDown [property]: [read-only] System.Reflection.MethodInfo list
 WoofWare.NUnitTestRunner.TestFixture.Tests [property]: [read-only] WoofWare.NUnitTestRunner.SingleTestMethod list
+WoofWare.NUnitTestRunner.TestFixture.Type [property]: [read-only] System.Type
 WoofWare.NUnitTestRunner.TestFixtureModule inherit obj
 WoofWare.NUnitTestRunner.TestFixtureModule.parse [static method]: System.Type -> WoofWare.NUnitTestRunner.TestFixture
-WoofWare.NUnitTestRunner.TestFixtureModule.run [static method]: WoofWare.NUnitTestRunner.ITestProgress -> (WoofWare.NUnitTestRunner.TestFixture -> WoofWare.NUnitTestRunner.SingleTestMethod -> bool) -> WoofWare.NUnitTestRunner.TestFixture -> WoofWare.NUnitTestRunner.FixtureRunResults
+WoofWare.NUnitTestRunner.TestFixtureModule.run [static method]: WoofWare.NUnitTestRunner.ITestProgress -> (WoofWare.NUnitTestRunner.TestFixture -> WoofWare.NUnitTestRunner.SingleTestMethod -> bool) -> WoofWare.NUnitTestRunner.TestFixture -> WoofWare.NUnitTestRunner.FixtureRunResults list
 WoofWare.NUnitTestRunner.TestKind inherit obj, implements WoofWare.NUnitTestRunner.TestKind System.IEquatable, System.Collections.IStructuralEquatable - union type with 3 cases
 WoofWare.NUnitTestRunner.TestKind+Data inherit WoofWare.NUnitTestRunner.TestKind
 WoofWare.NUnitTestRunner.TestKind+Data.get_Item [method]: unit -> obj list list

WoofWare.NUnitTestRunner.Lib/TestFixture.fs

@@ -1,6 +1,7 @@
 namespace WoofWare.NUnitTestRunner
 
 open System
+open System.Collections
 open System.Diagnostics
 open System.IO
 open System.Reflection
@@ -415,36 +416,15 @@ module TestFixture =
 
     /// Run every test (except those which fail the `filter`) in this test fixture, as well as the
     /// appropriate setup and tear-down logic.
-    let run
+    let private runOneFixture
         (progress : ITestProgress)
         (filter : TestFixture -> SingleTestMethod -> bool)
+        (name : string)
+        (containingObject : obj)
         (tests : TestFixture)
         : FixtureRunResults
         =
-        progress.OnTestFixtureStart tests.Name tests.Tests.Length
+        progress.OnTestFixtureStart name tests.Tests.Length
-
-        let containingObject =
-            let methods =
-                seq {
-                    match tests.OneTimeSetUp with
-                    | None -> ()
-                    | Some t -> yield t
-
-                    match tests.OneTimeTearDown with
-                    | None -> ()
-                    | Some t -> yield t
-
-                    yield! tests.Tests |> Seq.map (fun t -> t.Method)
-                }
-
-            methods
-            |> Seq.tryPick (fun mi ->
-                if not mi.IsStatic then
-                    Some (Activator.CreateInstance mi.DeclaringType)
-                else
-                    None
-            )
-            |> Option.toObj
 
         let oldWorkDir = Environment.CurrentDirectory
         Environment.CurrentDirectory <- FileInfo(tests.ContainingAssembly.Location).Directory.FullName
@@ -470,7 +450,7 @@ module TestFixture =
                 ExecutionId = Guid.NewGuid ()
                 TestId = Guid.NewGuid ()
                 // This one is a bit dubious, because we don't actually have a test name at all
-                TestName = tests.Name
+                TestName = name
                 ClassName = tests.Name
                 StdOut = if String.IsNullOrEmpty stdOut then None else Some stdOut
                 StdErr = if String.IsNullOrEmpty stdErr then None else Some stdErr
@@ -540,19 +520,27 @@ module TestFixture =
     /// Interpret this type as a [<TestFixture>], extracting the test members from it and annotating them with all
     /// relevant information about how we should run them.
     let parse (parentType : Type) : TestFixture =
-        if
+        let categories, args =
-            parentType.CustomAttributes
+            (([], []), parentType.CustomAttributes)
-            |> Seq.exists (fun attr -> attr.AttributeType.FullName = "NUnit.Framework.SetUpFixtureAttribute")
+            ||> Seq.fold (fun (categories, args) attr ->
-        then
+                match attr.AttributeType.FullName with
+                | "NUnit.Framework.SetUpFixtureAttribute" ->
                     failwith "This test runner does not support SetUpFixture. Please shout if you want this."
+                | "NUnit.Framework.CategoryAttribute" ->
+                    let cat = attr.ConstructorArguments |> Seq.exactlyOne |> _.Value |> unbox<string>
+                    cat :: categories, args
+                | "NUnit.Framework.TestFixtureAttribute" ->
+                    let newArgs =
+                        match attr.ConstructorArguments |> Seq.map _.Value |> Seq.toList with
+                        | [ :? ICollection as x ] ->
+                            x |> Seq.cast<CustomAttributeTypedArgument> |> Seq.map _.Value |> Seq.toList
+                        | xs -> xs
 
-        let categories =
+                    categories, newArgs :: args
-            parentType.CustomAttributes
+                | _ -> categories, args
-            |> Seq.filter (fun attr -> attr.AttributeType.FullName = "NUnit.Framework.CategoryAttribute")
+            )
-            |> Seq.map (fun attr -> attr.ConstructorArguments |> Seq.exactlyOne |> _.Value |> unbox<string>)
-            |> Seq.toList
 
-        (TestFixture.Empty parentType.Assembly parentType.Name, parentType.GetRuntimeMethods ())
+        (TestFixture.Empty parentType args, parentType.GetRuntimeMethods ())
         ||> Seq.fold (fun state mi ->
             ((state, []), mi.CustomAttributes)
             ||> Seq.fold (fun (state, unrecognisedAttrs) attr ->
@@ -622,3 +610,48 @@ module TestFixture =
 
                 state
         )
+
+    /// Run every test (except those which fail the `filter`) in this test fixture, as well as the
+    /// appropriate setup and tear-down logic.
+    let run
+        (progress : ITestProgress)
+        (filter : TestFixture -> SingleTestMethod -> bool)
+        (tests : TestFixture)
+        : FixtureRunResults list
+        =
+        match tests.Parameters with
+        | [] -> [ null ]
+        | args -> args |> List.map List.toArray
+        |> List.map (fun args ->
+            let containingObject =
+                let methods =
+                    seq {
+                        match tests.OneTimeSetUp with
+                        | None -> ()
+                        | Some t -> yield t
+
+                        match tests.OneTimeTearDown with
+                        | None -> ()
+                        | Some t -> yield t
+
+                        yield! tests.Tests |> Seq.map (fun t -> t.Method)
+                    }
+
+                methods
+                |> Seq.tryPick (fun mi ->
+                    if not mi.IsStatic then
+                        Some (Activator.CreateInstance (mi.DeclaringType, args))
+                    else
+                        None
+                )
+                |> Option.toObj
+
+            let name =
+                if isNull args then
+                    tests.Name
+                else
+                    let args = args |> Seq.map (fun o -> o.ToString ()) |> String.concat ","
+                    $"%s{tests.Name}(%s{args})"
+
+            runOneFixture progress filter name containingObject tests
+        )

WoofWare.NUnitTestRunner.Lib/version.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.8",
+  "version": "0.9",
   "publicReleaseRefSpec": [
     "^refs/heads/main$"
   ],

WoofWare.NUnitTestRunner/Program.fs

@@ -167,7 +167,7 @@ module Program =
         let testFixtures = assy.ExportedTypes |> Seq.map TestFixture.parse |> Seq.toList
 
         let creationTime = DateTimeOffset.Now
-        let results = testFixtures |> List.map (TestFixture.run progress filter)
+        let results = testFixtures |> List.collect (TestFixture.run progress filter)
 
         let finishTime = DateTimeOffset.Now
         let finishTimeHumanReadable = finishTime.ToString @"yyyy-MM-dd HH:mm:ss"

WoofWare.NUnitTestRunner/WoofWare.NUnitTestRunner.Test/TestList.fs

@@ -11,6 +11,8 @@ module TestList =
     [<Test>]
     let ``combinations has right size`` () =
         let property (xs : int list list) =
+            let xs = if xs.Length > 6 then xs |> List.take 6 else xs
+            let xs = xs |> List.map (fun xs -> if xs.Length > 6 then xs |> List.take 6 else xs)
             let combs = List.combinations xs
 
             combs.Length