Generalise Spectre to arbitrary console (#78)

Consumer/Consumer.fsproj

@@ -10,6 +10,9 @@
     <ItemGroup>
         <Compile Include="NoAttribute.fs" />
         <Compile Include="Inconclusive.fs" />
+        <Compile Include="TestParallel.fs" />
+        <Compile Include="TestStdout.fs" />
+        <Compile Include="TestParameterisedFixture.fs" />
         <Compile Include="TestSetUp.fs" />
         <Compile Include="TestValues.fs" />
         <None Include="some-config.json">

Consumer/TestCaseData.fs

@@ -35,6 +35,13 @@ module TestCaseData =
     [<TestCaseSource(nameof optionalRaw)>]
     let ``Consume options, raw`` (s : string option) : unit = s |> shouldEqual s
 
+    [<TestCase(30, 15, 44, false)>]
+    let bug66 (i : int, j : int, k : int, l : bool) =
+        i |> shouldEqual 30
+        j |> shouldEqual 15
+        k |> shouldEqual 44
+        l |> shouldEqual false
+
     [<OneTimeTearDown>]
     let tearDown () =
         testCasesSeen

Consumer/TestParallel.fs

@@ -0,0 +1,49 @@
+namespace Consumer
+
+open NUnit.Framework
+open FsUnitTyped
+
+[<TestFixture>]
+[<Parallelizable>]
+module TestParallelDefault =
+
+    let defaults = List.init 100 id
+
+    [<TestCaseSource(nameof defaults)>]
+    let ``Default thing`` (i : int) = i |> shouldEqual i
+
+[<TestFixture>]
+[<Parallelizable(ParallelScope.All)>]
+module TestParallelAllScope =
+
+    let defaults = List.init 100 id
+
+    [<TestCaseSource(nameof defaults)>]
+    let ``Default thing`` (i : int) = i |> shouldEqual i
+
+[<TestFixture>]
+[<Parallelizable(ParallelScope.Self)>]
+module TestParallelSelfScope =
+
+    let defaults = List.init 100 id
+
+    [<TestCaseSource(nameof defaults)>]
+    let ``Default thing`` (i : int) = i |> shouldEqual i
+
+[<TestFixture>]
+[<Parallelizable(ParallelScope.Children)>]
+module TestParallelChildrenScope =
+
+    let defaults = List.init 100 id
+
+    [<TestCaseSource(nameof defaults)>]
+    let ``Default thing`` (i : int) = i |> shouldEqual i
+
+[<TestFixture>]
+[<Parallelizable(ParallelScope.Fixtures)>]
+module TestParallelFixturesScope =
+
+    let defaults = List.init 100 id
+
+    [<TestCaseSource(nameof defaults)>]
+    let ``Default thing`` (i : int) = i |> shouldEqual i

Consumer/TestParameterisedFixture.fs

@@ -0,0 +1,18 @@
+namespace Consumer
+
+open NUnit.Framework
+open FsUnitTyped
+
+[<TestFixture true>]
+[<TestFixture false>]
+type TestParameterisedFixture (v : bool) =
+    [<Test>]
+    member _.Thing () = v |> shouldEqual v
+
+[<TestFixture(3, true)>]
+[<TestFixture(6, false)>]
+type TestParameterisedFixtureMultiple (i : int, v : bool) =
+    [<Test>]
+    member _.Thing () =
+        v |> shouldEqual v
+        i |> shouldEqual i

Consumer/TestStdout.fs

@@ -0,0 +1,13 @@
+namespace Consumer
+
+open System
+open NUnit.Framework
+
+[<TestFixture>]
+module TestStdout =
+
+    [<Test>]
+    let ``Stdout is redirected`` () =
+        Console.Out.WriteLine "Hi!"
+        Console.WriteLine "Hi! part 2"
+        Console.Error.WriteLine "Bye!"

README.md

@@ -8,3 +8,9 @@ To supply special characters in a string, XML-encode them and `"quote"` the stri
 We support at least the [documented `dotnet test` examples](https://learn.microsoft.com/en-us/dotnet/core/testing/selective-unit-tests).
 However, we would recommend phrasing some of them differently, for maximum peace of mind:
 * `FullyQualifiedName=MyNamespace.MyTestsClass<ParameterType1%2CParameterType2>.MyTestMethod`. This would be better phrased with quotes and escaping as `FullyQualifiedName="MyNamespace.MyTestsClass&lt;ParameterType1%2CParameterType2&gt;.MyTestMethod"`
+
+## Parallelism
+
+WoofWare.NUnitTestRunner has *limited* support for parallelism.
+By default, we run tests serially; we may or may not respect the NUnit parallelism attributes to any given extent (but we will never incorrectly run tests in parallel).
+For example, as of this writing, we do not run any tests in parallel (but the internal infrastructure is set up so that we will be able to do this soon).

WoofWare.NUnitTestRunner.Lib/Domain.fs

@@ -1,5 +1,6 @@
 namespace WoofWare.NUnitTestRunner
 
+open System
 open System.Reflection
 
 /// A modifier on whether a given test should be run.
@@ -30,6 +31,36 @@ type Combinatorial =
     /// each", and so on. Spare slots are filled with `Unchecked.defaultof<_>`.
     | Sequential
 
+/// Describes the level of parallelism permitted in some context.
+[<RequireQualifiedAccess>]
+type ClassParallelScope =
+    /// "I may be run in parallel with other tests, although my children might not be able to run in parallel with each
+    /// other".
+    | Self
+    /// "The set of things I contain may be run in parallel with itself".
+    | Children
+    /// "Fixtures within me may be run in parallel with each other, but the tests within a given fixture might not
+    /// be runnable in parallel with each other".
+    | Fixtures
+    /// "All my descendents are happy to run in parallel with anything else, and also so am I".
+    | All
+
+/// Describes the level of parallelism permitted within an assembly.
+[<RequireQualifiedAccess>]
+type AssemblyParallelScope =
+    /// "The set of things I contain may be run in parallel with itself".
+    | Children
+    /// "Fixtures within me may be run in parallel with each other, but the tests within a given fixture might not
+    /// necessarily be runnable in parallel with each other".
+    | Fixtures
+
+/// Describes whether a test can be run concurrently with other tests.
+type Parallelizable<'scope> =
+    /// This test is happy, under some conditions (specified by the scope), to be run alongside other tests.
+    | Yes of 'scope
+    /// This test must always be run on its own.
+    | No
+
 /// A single method or member which holds some tests. (Often such a member will represent only one test, but e.g.
 /// if it has [<TestCaseSource>] then it represents multiple tests.)
 type SingleTestMethod =
@@ -48,6 +79,8 @@ type SingleTestMethod =
         /// If this test has data supplied by `[<Value>]` annotations, specifies how those annotations are combined
         /// to produce the complete collection of args.
         Combinatorial : Combinatorial option
+        /// If this test has declared a parallelisability, that goes here.
+        Parallelize : Parallelizable<unit> option
     }
 
     /// Human-readable name of this test method.
@@ -62,6 +95,8 @@ type TestFixture =
         /// Fully-qualified name of this fixture (e.g. MyThing.Test.Foo for `[<TestFixture>] module Foo` in the
         /// `MyThing.Test` assembly).
         Name : string
+        /// The type which is this fixture, containing the tests as members.
+        Type : Type
         /// A method which is run once when this test fixture starts, before any other setup logic and before
         /// any tests run. If this method fails, no tests will run and no per-test setup/teardown logic will run,
         /// but OneTimeTearDown will run.
@@ -77,20 +112,28 @@ type TestFixture =
         /// Methods which are run in some arbitrary order after each individual test, even if the test or its setup
         /// failed. If the first TearDown we run fails, we don't define whether the other TearDowns run.
         TearDown : MethodInfo list
+        /// You might have defined e.g. `[<TestFixture true>] type Foo (v : bool) = ...`. If so, this gives the
+        /// various possible parameters.
+        Parameters : obj list list
         /// The individual test methods present within this fixture.
         Tests : SingleTestMethod list
+        /// If this fixture has declared a parallelisability, that goes here.
+        Parallelize : Parallelizable<ClassParallelScope> option
     }
 
     /// A test fixture about which we know nothing. No tests, no setup/teardown.
-    static member Empty (containingAssembly : Assembly) (name : string) =
+    static member Empty (ty : Type) (par : Parallelizable<ClassParallelScope> option) (args : obj list list) =
         {
-            ContainingAssembly = containingAssembly
+            ContainingAssembly = ty.Assembly
-            Name = name
+            Type = ty
+            Name = ty.Name
             OneTimeSetUp = None
             OneTimeTearDown = None
             SetUp = []
             TearDown = []
+            Parameters = args
             Tests = []
+            Parallelize = par
         }
 
 /// User code in the unit under test has failed somehow.

WoofWare.NUnitTestRunner.Lib/DotnetRuntime.fs

@@ -0,0 +1,104 @@
+namespace WoofWare.NUnitTestRunner
+
+open System
+open System.IO
+open WoofWare.DotnetRuntimeLocator
+
+/// Functions for locating .NET runtimes.
+[<RequireQualifiedAccess>]
+module DotnetRuntime =
+    let private selectRuntime
+        (config : RuntimeOptions)
+        (f : DotnetEnvironmentInfo)
+        : Choice<DotnetEnvironmentFrameworkInfo, DotnetEnvironmentSdkInfo> option
+        =
+        let rollForward =
+            match Environment.GetEnvironmentVariable "DOTNET_ROLL_FORWARD" with
+            | null ->
+                config.RollForward
+                |> Option.map RollForward.Parse
+                |> Option.defaultValue RollForward.Minor
+            | s -> RollForward.Parse s
+
+        let desiredVersions =
+            match config.Framework with
+            | Some f -> [ Version f.Version, f.Name ]
+            | None ->
+
+            match config.Frameworks with
+            | Some f -> f |> List.map (fun f -> Version f.Version, f.Name)
+            | None ->
+                failwith
+                    "Could not deduce a framework version due to lack of either Framework or Frameworks in runtimeconfig"
+
+        let compatiblyNamedRuntimes =
+            f.Frameworks
+            |> Seq.collect (fun availableFramework ->
+                desiredVersions
+                |> List.choose (fun (desiredVersion, desiredName) ->
+                    if desiredName = availableFramework.Name then
+                        Some
+                            {|
+                                Desired = desiredVersion
+                                Name = desiredName
+                                Installed = availableFramework
+                                InstalledVersion = Version availableFramework.Version
+                            |}
+                    else
+                        None
+                )
+            )
+            |> Seq.toList
+
+        match rollForward with
+        | RollForward.Minor ->
+            let available =
+                compatiblyNamedRuntimes
+                |> Seq.filter (fun data ->
+                    data.InstalledVersion.Major = data.Desired.Major
+                    && data.InstalledVersion.Minor >= data.Desired.Minor
+                )
+                |> Seq.groupBy (fun data -> data.Name)
+                |> Seq.map (fun (name, data) ->
+                    let data =
+                        data
+                        |> Seq.minBy (fun data -> data.InstalledVersion.Minor, data.InstalledVersion.Build)
+
+                    name, data.Installed
+                )
+                // TODO: how do we select between many available frameworks?
+                |> Seq.tryHead
+
+            match available with
+            | Some (_, f) -> Some (Choice1Of2 f)
+            | None ->
+                // TODO: maybe we can ask the SDK. But we keep on trucking: maybe we're self-contained,
+                // and we'll actually find all the runtime next to the DLL.
+                None
+        | _ -> failwith "non-minor RollForward not supported yet; please shout if you want it"
+
+    /// Given an executable DLL, locate the .NET runtime that can best run it.
+    let locate (dll : FileInfo) : DirectoryInfo list =
+        let runtimeConfig =
+            let name =
+                if not (dll.Name.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) then
+                    failwith $"Expected DLL %s{dll.FullName} to end in .dll"
+
+                dll.Name.Substring (0, dll.Name.Length - 4)
+
+            Path.Combine (dll.Directory.FullName, $"%s{name}.runtimeconfig.json")
+            |> File.ReadAllText
+            |> System.Text.Json.Nodes.JsonNode.Parse
+            |> RuntimeConfig.jsonParse
+            |> fun f -> f.RuntimeOptions
+
+        let availableRuntimes = DotnetEnvironmentInfo.Get ()
+
+        let runtime = selectRuntime runtimeConfig availableRuntimes
+
+        match runtime with
+        | None ->
+            // Keep on trucking: let's be optimistic and hope that we're self-contained.
+            [ dll.Directory ]
+        | Some (Choice1Of2 runtime) -> [ dll.Directory ; DirectoryInfo $"%s{runtime.Path}/%s{runtime.Version}" ]
+        | Some (Choice2Of2 sdk) -> [ dll.Directory ; DirectoryInfo sdk.Path ]

WoofWare.NUnitTestRunner.Lib/RuntimeConfig.fs

@@ -4,14 +4,14 @@ open System
 open WoofWare.Myriad.Plugins
 
 [<JsonParse>]
-type FrameworkDescription =
+type internal FrameworkDescription =
     {
         Name : string
         Version : string
     }
 
 [<JsonParse>]
-type RuntimeOptions =
+type internal RuntimeOptions =
     {
         Tfm : string
         Framework : FrameworkDescription option
@@ -20,13 +20,13 @@ type RuntimeOptions =
     }
 
 [<JsonParse>]
-type RuntimeConfig =
+type internal RuntimeConfig =
     {
         RuntimeOptions : RuntimeOptions
     }
 
 [<RequireQualifiedAccess>]
-type RollForward =
+type internal RollForward =
     | Minor
     | Major
     | LatestPatch

WoofWare.NUnitTestRunner.Lib/SingleTestMethod.fs

@@ -18,75 +18,95 @@ module SingleTestMethod =
         (attrs : CustomAttributeData list)
         : SingleTestMethod option * CustomAttributeData list
         =
-        let remaining, isTest, sources, hasData, modifiers, categories, repeat, comb =
+        let remaining, isTest, sources, hasData, modifiers, categories, repeat, comb, par =
-            (([], false, [], None, [], [], None, None), attrs)
+            (([], false, [], None, [], [], None, None, None), attrs)
-            ||> List.fold (fun (remaining, isTest, sources, hasData, mods, cats, repeat, comb) attr ->
+            ||> List.fold (fun (remaining, isTest, sources, hasData, mods, cats, repeat, comb, par) attr ->
                 match attr.AttributeType.FullName with
                 | "NUnit.Framework.TestAttribute" ->
                     if attr.ConstructorArguments.Count > 0 then
                         failwith "Unexpectedly got arguments to the Test attribute"
 
-                    (remaining, true, sources, hasData, mods, cats, repeat, comb)
+                    (remaining, true, sources, hasData, mods, cats, repeat, comb, par)
                 | "NUnit.Framework.TestCaseAttribute" ->
                     let args = attr.ConstructorArguments |> Seq.map _.Value |> Seq.toList
 
+                    let args =
+                        match args with
+                        | [ :? System.Collections.ICollection as x ] ->
+                            x
+                            |> Seq.cast<CustomAttributeTypedArgument>
+                            |> Seq.map (fun v -> v.Value)
+                            |> Seq.toList
+                        | _ -> args
+
                     match hasData with
-                    | None -> (remaining, isTest, sources, Some [ List.ofSeq args ], mods, cats, repeat, comb)
+                    | None -> (remaining, isTest, sources, Some [ List.ofSeq args ], mods, cats, repeat, comb, par)
                     | Some existing ->
-                        (remaining, isTest, sources, Some ((List.ofSeq args) :: existing), mods, cats, repeat, comb)
+                        let args = (List.ofSeq args) :: existing |> Some
+                        (remaining, isTest, sources, args, mods, cats, repeat, comb, par)
                 | "NUnit.Framework.TestCaseSourceAttribute" ->
                     let arg = attr.ConstructorArguments |> Seq.exactlyOne |> _.Value |> unbox<string>
 
-                    (remaining, isTest, arg :: sources, hasData, mods, cats, repeat, comb)
+                    (remaining, isTest, arg :: sources, hasData, mods, cats, repeat, comb, par)
                 | "NUnit.Framework.ExplicitAttribute" ->
                     let reason =
                         attr.ConstructorArguments
                         |> Seq.tryHead
                         |> Option.map (_.Value >> unbox<string>)
 
-                    (remaining, isTest, sources, hasData, (Modifier.Explicit reason) :: mods, cats, repeat, comb)
+                    (remaining, isTest, sources, hasData, (Modifier.Explicit reason) :: mods, cats, repeat, comb, par)
                 | "NUnit.Framework.IgnoreAttribute" ->
                     let reason =
                         attr.ConstructorArguments
                         |> Seq.tryHead
                         |> Option.map (_.Value >> unbox<string>)
 
-                    (remaining, isTest, sources, hasData, (Modifier.Ignored reason) :: mods, cats, repeat, comb)
+                    (remaining, isTest, sources, hasData, (Modifier.Ignored reason) :: mods, cats, repeat, comb, par)
                 | "NUnit.Framework.CategoryAttribute" ->
                     let category =
                         attr.ConstructorArguments |> Seq.exactlyOne |> _.Value |> unbox<string>
 
-                    (remaining, isTest, sources, hasData, mods, category :: cats, repeat, comb)
+                    (remaining, isTest, sources, hasData, mods, category :: cats, repeat, comb, par)
                 | "NUnit.Framework.RepeatAttribute" ->
                     match repeat with
                     | Some _ -> failwith $"Got RepeatAttribute multiple times on %s{method.Name}"
                     | None ->
 
                     let repeat = attr.ConstructorArguments |> Seq.exactlyOne |> _.Value |> unbox<int>
-                    (remaining, isTest, sources, hasData, mods, cats, Some repeat, comb)
+                    (remaining, isTest, sources, hasData, mods, cats, Some repeat, comb, par)
                 | "NUnit.Framework.CombinatorialAttribute" ->
                     match comb with
                     | Some _ ->
                         failwith $"Got CombinatorialAttribute or SequentialAttribute multiple times on %s{method.Name}"
                     | None ->
-                        (remaining, isTest, sources, hasData, mods, cats, repeat, Some Combinatorial.Combinatorial)
+                        (remaining, isTest, sources, hasData, mods, cats, repeat, Some Combinatorial.Combinatorial, par)
                 | "NUnit.Framework.SequentialAttribute" ->
                     match comb with
                     | Some _ ->
                         failwith $"Got CombinatorialAttribute or SequentialAttribute multiple times on %s{method.Name}"
-                    | None -> (remaining, isTest, sources, hasData, mods, cats, repeat, Some Combinatorial.Sequential)
+                    | None ->
+                        (remaining, isTest, sources, hasData, mods, cats, repeat, Some Combinatorial.Sequential, par)
+                | "NUnit.Framework.NonParallelizableAttribute" ->
+                    match par with
+                    | Some _ -> failwith $"Got a parallelization attribute multiple times on %s{method.Name}"
+                    | None -> (remaining, isTest, sources, hasData, mods, cats, repeat, comb, Some Parallelizable.No)
+                | "NUnit.Framework.ParallelizableAttribute" ->
+                    match par with
+                    | Some _ -> failwith $"Got multiple parallelization attributes on %s{method.Name}"
+                    | None ->
+                        (remaining, isTest, sources, hasData, mods, cats, repeat, comb, Some (Parallelizable.Yes ()))
                 | s when s.StartsWith ("NUnit.Framework", StringComparison.Ordinal) ->
                     failwith $"Unrecognised attribute on function %s{method.Name}: %s{attr.AttributeType.FullName}"
-                | _ -> (attr :: remaining, isTest, sources, hasData, mods, cats, repeat, comb)
+                | _ -> (attr :: remaining, isTest, sources, hasData, mods, cats, repeat, comb, par)
             )
 
         let test =
-            match isTest, sources, hasData, modifiers, categories, repeat, comb with
+            match isTest, sources, hasData, modifiers, categories, repeat, comb, par with
-            | _, _ :: _, Some _, _, _, _, _ ->
+            | _, _ :: _, Some _, _, _, _, _, _ ->
                 failwith
                     $"Test '%s{method.Name}' unexpectedly has both TestData and TestCaseSource; not currently supported"
-            | false, [], None, [], _, _, _ -> None
+            | false, [], None, [], _, _, _, _ -> None
-            | _, _ :: _, None, mods, categories, repeat, comb ->
+            | _, _ :: _, None, mods, categories, repeat, comb, par ->
                 {
                     Kind = TestKind.Source sources
                     Method = method
@@ -94,9 +114,10 @@ module SingleTestMethod =
                     Categories = categories @ parentCategories
                     Repeat = repeat
                     Combinatorial = comb
+                    Parallelize = par
                 }
                 |> Some
-            | _, [], Some data, mods, categories, repeat, comb ->
+            | _, [], Some data, mods, categories, repeat, comb, par ->
                 {
                     Kind = TestKind.Data data
                     Method = method
@@ -104,9 +125,10 @@ module SingleTestMethod =
                     Categories = categories @ parentCategories
                     Repeat = repeat
                     Combinatorial = comb
+                    Parallelize = par
                 }
                 |> Some
-            | true, [], None, mods, categories, repeat, comb ->
+            | true, [], None, mods, categories, repeat, comb, par ->
                 {
                     Kind = TestKind.Single
                     Method = method
@@ -114,9 +136,10 @@ module SingleTestMethod =
                     Categories = categories @ parentCategories
                     Repeat = repeat
                     Combinatorial = comb
+                    Parallelize = par
                 }
                 |> Some
-            | false, [], None, _ :: _, _, _, _ ->
+            | false, [], None, _ :: _, _, _, _, _ ->
                 failwith
                     $"Unexpectedly got test modifiers but no test settings on '%s{method.Name}', which you probably didn't intend."
 

WoofWare.NUnitTestRunner.Lib/SurfaceBaseline.txt

@@ -1,3 +1,41 @@
+WoofWare.NUnitTestRunner.AssemblyParallelScope inherit obj, implements WoofWare.NUnitTestRunner.AssemblyParallelScope System.IEquatable, System.Collections.IStructuralEquatable, WoofWare.NUnitTestRunner.AssemblyParallelScope System.IComparable, System.IComparable, System.Collections.IStructuralComparable - union type with 2 cases
+WoofWare.NUnitTestRunner.AssemblyParallelScope+Tags inherit obj
+WoofWare.NUnitTestRunner.AssemblyParallelScope+Tags.Children [static field]: int = 0
+WoofWare.NUnitTestRunner.AssemblyParallelScope+Tags.Fixtures [static field]: int = 1
+WoofWare.NUnitTestRunner.AssemblyParallelScope.Children [static property]: [read-only] WoofWare.NUnitTestRunner.AssemblyParallelScope
+WoofWare.NUnitTestRunner.AssemblyParallelScope.Fixtures [static property]: [read-only] WoofWare.NUnitTestRunner.AssemblyParallelScope
+WoofWare.NUnitTestRunner.AssemblyParallelScope.get_Children [static method]: unit -> WoofWare.NUnitTestRunner.AssemblyParallelScope
+WoofWare.NUnitTestRunner.AssemblyParallelScope.get_Fixtures [static method]: unit -> WoofWare.NUnitTestRunner.AssemblyParallelScope
+WoofWare.NUnitTestRunner.AssemblyParallelScope.get_IsChildren [method]: unit -> bool
+WoofWare.NUnitTestRunner.AssemblyParallelScope.get_IsFixtures [method]: unit -> bool
+WoofWare.NUnitTestRunner.AssemblyParallelScope.get_Tag [method]: unit -> int
+WoofWare.NUnitTestRunner.AssemblyParallelScope.IsChildren [property]: [read-only] bool
+WoofWare.NUnitTestRunner.AssemblyParallelScope.IsFixtures [property]: [read-only] bool
+WoofWare.NUnitTestRunner.AssemblyParallelScope.Tag [property]: [read-only] int
+WoofWare.NUnitTestRunner.ClassParallelScope inherit obj, implements WoofWare.NUnitTestRunner.ClassParallelScope System.IEquatable, System.Collections.IStructuralEquatable, WoofWare.NUnitTestRunner.ClassParallelScope System.IComparable, System.IComparable, System.Collections.IStructuralComparable - union type with 4 cases
+WoofWare.NUnitTestRunner.ClassParallelScope+Tags inherit obj
+WoofWare.NUnitTestRunner.ClassParallelScope+Tags.All [static field]: int = 3
+WoofWare.NUnitTestRunner.ClassParallelScope+Tags.Children [static field]: int = 1
+WoofWare.NUnitTestRunner.ClassParallelScope+Tags.Fixtures [static field]: int = 2
+WoofWare.NUnitTestRunner.ClassParallelScope+Tags.Self [static field]: int = 0
+WoofWare.NUnitTestRunner.ClassParallelScope.All [static property]: [read-only] WoofWare.NUnitTestRunner.ClassParallelScope
+WoofWare.NUnitTestRunner.ClassParallelScope.Children [static property]: [read-only] WoofWare.NUnitTestRunner.ClassParallelScope
+WoofWare.NUnitTestRunner.ClassParallelScope.Fixtures [static property]: [read-only] WoofWare.NUnitTestRunner.ClassParallelScope
+WoofWare.NUnitTestRunner.ClassParallelScope.get_All [static method]: unit -> WoofWare.NUnitTestRunner.ClassParallelScope
+WoofWare.NUnitTestRunner.ClassParallelScope.get_Children [static method]: unit -> WoofWare.NUnitTestRunner.ClassParallelScope
+WoofWare.NUnitTestRunner.ClassParallelScope.get_Fixtures [static method]: unit -> WoofWare.NUnitTestRunner.ClassParallelScope
+WoofWare.NUnitTestRunner.ClassParallelScope.get_IsAll [method]: unit -> bool
+WoofWare.NUnitTestRunner.ClassParallelScope.get_IsChildren [method]: unit -> bool
+WoofWare.NUnitTestRunner.ClassParallelScope.get_IsFixtures [method]: unit -> bool
+WoofWare.NUnitTestRunner.ClassParallelScope.get_IsSelf [method]: unit -> bool
+WoofWare.NUnitTestRunner.ClassParallelScope.get_Self [static method]: unit -> WoofWare.NUnitTestRunner.ClassParallelScope
+WoofWare.NUnitTestRunner.ClassParallelScope.get_Tag [method]: unit -> int
+WoofWare.NUnitTestRunner.ClassParallelScope.IsAll [property]: [read-only] bool
+WoofWare.NUnitTestRunner.ClassParallelScope.IsChildren [property]: [read-only] bool
+WoofWare.NUnitTestRunner.ClassParallelScope.IsFixtures [property]: [read-only] bool
+WoofWare.NUnitTestRunner.ClassParallelScope.IsSelf [property]: [read-only] bool
+WoofWare.NUnitTestRunner.ClassParallelScope.Self [static property]: [read-only] WoofWare.NUnitTestRunner.ClassParallelScope
+WoofWare.NUnitTestRunner.ClassParallelScope.Tag [property]: [read-only] int
 WoofWare.NUnitTestRunner.Combinatorial inherit obj, implements WoofWare.NUnitTestRunner.Combinatorial System.IEquatable, System.Collections.IStructuralEquatable, WoofWare.NUnitTestRunner.Combinatorial System.IComparable, System.IComparable, System.Collections.IStructuralComparable - union type with 2 cases
 WoofWare.NUnitTestRunner.Combinatorial+Tags inherit obj
 WoofWare.NUnitTestRunner.Combinatorial+Tags.Combinatorial [static field]: int = 0
@@ -12,6 +50,8 @@ WoofWare.NUnitTestRunner.Combinatorial.IsCombinatorial [property]: [read-only] b
 WoofWare.NUnitTestRunner.Combinatorial.IsSequential [property]: [read-only] bool
 WoofWare.NUnitTestRunner.Combinatorial.Sequential [static property]: [read-only] WoofWare.NUnitTestRunner.Combinatorial
 WoofWare.NUnitTestRunner.Combinatorial.Tag [property]: [read-only] int
+WoofWare.NUnitTestRunner.DotnetRuntime inherit obj
+WoofWare.NUnitTestRunner.DotnetRuntime.locate [static method]: System.IO.FileInfo -> System.IO.DirectoryInfo list
 WoofWare.NUnitTestRunner.Filter inherit obj, implements WoofWare.NUnitTestRunner.Filter System.IEquatable, System.Collections.IStructuralEquatable, WoofWare.NUnitTestRunner.Filter System.IComparable, System.IComparable, System.Collections.IStructuralComparable - union type with 6 cases
 WoofWare.NUnitTestRunner.Filter+And inherit WoofWare.NUnitTestRunner.Filter
 WoofWare.NUnitTestRunner.Filter+And.get_Item1 [method]: unit -> WoofWare.NUnitTestRunner.Filter
@@ -139,8 +179,24 @@ WoofWare.NUnitTestRunner.Modifier.IsIgnored [property]: [read-only] bool
 WoofWare.NUnitTestRunner.Modifier.NewExplicit [static method]: string option -> WoofWare.NUnitTestRunner.Modifier
 WoofWare.NUnitTestRunner.Modifier.NewIgnored [static method]: string option -> WoofWare.NUnitTestRunner.Modifier
 WoofWare.NUnitTestRunner.Modifier.Tag [property]: [read-only] int
+WoofWare.NUnitTestRunner.Parallelizable`1 inherit obj, implements 'scope WoofWare.NUnitTestRunner.Parallelizable System.IEquatable, System.Collections.IStructuralEquatable, 'scope WoofWare.NUnitTestRunner.Parallelizable System.IComparable, System.IComparable, System.Collections.IStructuralComparable - union type with 2 cases
+WoofWare.NUnitTestRunner.Parallelizable`1+Tags inherit obj
+WoofWare.NUnitTestRunner.Parallelizable`1+Tags.No [static field]: int = 1
+WoofWare.NUnitTestRunner.Parallelizable`1+Tags.Yes [static field]: int = 0
+WoofWare.NUnitTestRunner.Parallelizable`1+Yes inherit 'scope WoofWare.NUnitTestRunner.Parallelizable
+WoofWare.NUnitTestRunner.Parallelizable`1+Yes.get_Item [method]: unit -> 'scope
+WoofWare.NUnitTestRunner.Parallelizable`1+Yes.Item [property]: [read-only] 'scope
+WoofWare.NUnitTestRunner.Parallelizable`1.get_IsNo [method]: unit -> bool
+WoofWare.NUnitTestRunner.Parallelizable`1.get_IsYes [method]: unit -> bool
+WoofWare.NUnitTestRunner.Parallelizable`1.get_No [static method]: unit -> 'scope WoofWare.NUnitTestRunner.Parallelizable
+WoofWare.NUnitTestRunner.Parallelizable`1.get_Tag [method]: unit -> int
+WoofWare.NUnitTestRunner.Parallelizable`1.IsNo [property]: [read-only] bool
+WoofWare.NUnitTestRunner.Parallelizable`1.IsYes [property]: [read-only] bool
+WoofWare.NUnitTestRunner.Parallelizable`1.NewYes [static method]: 'scope -> 'scope WoofWare.NUnitTestRunner.Parallelizable
+WoofWare.NUnitTestRunner.Parallelizable`1.No [static property]: [read-only] 'scope WoofWare.NUnitTestRunner.Parallelizable
+WoofWare.NUnitTestRunner.Parallelizable`1.Tag [property]: [read-only] int
 WoofWare.NUnitTestRunner.SingleTestMethod inherit obj, implements WoofWare.NUnitTestRunner.SingleTestMethod System.IEquatable, System.Collections.IStructuralEquatable
-WoofWare.NUnitTestRunner.SingleTestMethod..ctor [constructor]: (System.Reflection.MethodInfo, WoofWare.NUnitTestRunner.TestKind, WoofWare.NUnitTestRunner.Modifier list, string list, int option, WoofWare.NUnitTestRunner.Combinatorial option)
+WoofWare.NUnitTestRunner.SingleTestMethod..ctor [constructor]: (System.Reflection.MethodInfo, WoofWare.NUnitTestRunner.TestKind, WoofWare.NUnitTestRunner.Modifier list, string list, int option, WoofWare.NUnitTestRunner.Combinatorial option, unit WoofWare.NUnitTestRunner.Parallelizable option)
 WoofWare.NUnitTestRunner.SingleTestMethod.Categories [property]: [read-only] string list
 WoofWare.NUnitTestRunner.SingleTestMethod.Combinatorial [property]: [read-only] WoofWare.NUnitTestRunner.Combinatorial option
 WoofWare.NUnitTestRunner.SingleTestMethod.get_Categories [method]: unit -> string list
@@ -149,11 +205,13 @@ WoofWare.NUnitTestRunner.SingleTestMethod.get_Kind [method]: unit -> WoofWare.NU
 WoofWare.NUnitTestRunner.SingleTestMethod.get_Method [method]: unit -> System.Reflection.MethodInfo
 WoofWare.NUnitTestRunner.SingleTestMethod.get_Modifiers [method]: unit -> WoofWare.NUnitTestRunner.Modifier list
 WoofWare.NUnitTestRunner.SingleTestMethod.get_Name [method]: unit -> string
+WoofWare.NUnitTestRunner.SingleTestMethod.get_Parallelize [method]: unit -> unit WoofWare.NUnitTestRunner.Parallelizable option
 WoofWare.NUnitTestRunner.SingleTestMethod.get_Repeat [method]: unit -> int option
 WoofWare.NUnitTestRunner.SingleTestMethod.Kind [property]: [read-only] WoofWare.NUnitTestRunner.TestKind
 WoofWare.NUnitTestRunner.SingleTestMethod.Method [property]: [read-only] System.Reflection.MethodInfo
 WoofWare.NUnitTestRunner.SingleTestMethod.Modifiers [property]: [read-only] WoofWare.NUnitTestRunner.Modifier list
 WoofWare.NUnitTestRunner.SingleTestMethod.Name [property]: [read-only] string
+WoofWare.NUnitTestRunner.SingleTestMethod.Parallelize [property]: [read-only] unit WoofWare.NUnitTestRunner.Parallelizable option
 WoofWare.NUnitTestRunner.SingleTestMethod.Repeat [property]: [read-only] int option
 WoofWare.NUnitTestRunner.SingleTestMethodModule inherit obj
 WoofWare.NUnitTestRunner.SingleTestMethodModule.parse [static method]: string list -> System.Reflection.MethodInfo -> System.Reflection.CustomAttributeData list -> (WoofWare.NUnitTestRunner.SingleTestMethod option * System.Reflection.CustomAttributeData list)
@@ -185,25 +243,31 @@ WoofWare.NUnitTestRunner.TestFailure.NewTearDownFailed [static method]: WoofWare
 WoofWare.NUnitTestRunner.TestFailure.NewTestFailed [static method]: WoofWare.NUnitTestRunner.UserMethodFailure -> WoofWare.NUnitTestRunner.TestFailure
 WoofWare.NUnitTestRunner.TestFailure.Tag [property]: [read-only] int
 WoofWare.NUnitTestRunner.TestFixture inherit obj, implements WoofWare.NUnitTestRunner.TestFixture System.IEquatable, System.Collections.IStructuralEquatable
-WoofWare.NUnitTestRunner.TestFixture..ctor [constructor]: (System.Reflection.Assembly, string, System.Reflection.MethodInfo option, System.Reflection.MethodInfo option, System.Reflection.MethodInfo list, System.Reflection.MethodInfo list, WoofWare.NUnitTestRunner.SingleTestMethod list)
+WoofWare.NUnitTestRunner.TestFixture..ctor [constructor]: (System.Reflection.Assembly, string, System.Type, System.Reflection.MethodInfo option, System.Reflection.MethodInfo option, System.Reflection.MethodInfo list, System.Reflection.MethodInfo list, obj list list, WoofWare.NUnitTestRunner.SingleTestMethod list, WoofWare.NUnitTestRunner.ClassParallelScope WoofWare.NUnitTestRunner.Parallelizable option)
 WoofWare.NUnitTestRunner.TestFixture.ContainingAssembly [property]: [read-only] System.Reflection.Assembly
-WoofWare.NUnitTestRunner.TestFixture.Empty [static method]: System.Reflection.Assembly -> string -> WoofWare.NUnitTestRunner.TestFixture
+WoofWare.NUnitTestRunner.TestFixture.Empty [static method]: System.Type -> WoofWare.NUnitTestRunner.ClassParallelScope WoofWare.NUnitTestRunner.Parallelizable option -> obj list list -> WoofWare.NUnitTestRunner.TestFixture
 WoofWare.NUnitTestRunner.TestFixture.get_ContainingAssembly [method]: unit -> System.Reflection.Assembly
 WoofWare.NUnitTestRunner.TestFixture.get_Name [method]: unit -> string
 WoofWare.NUnitTestRunner.TestFixture.get_OneTimeSetUp [method]: unit -> System.Reflection.MethodInfo option
 WoofWare.NUnitTestRunner.TestFixture.get_OneTimeTearDown [method]: unit -> System.Reflection.MethodInfo option
+WoofWare.NUnitTestRunner.TestFixture.get_Parallelize [method]: unit -> WoofWare.NUnitTestRunner.ClassParallelScope WoofWare.NUnitTestRunner.Parallelizable option
+WoofWare.NUnitTestRunner.TestFixture.get_Parameters [method]: unit -> obj list list
 WoofWare.NUnitTestRunner.TestFixture.get_SetUp [method]: unit -> System.Reflection.MethodInfo list
 WoofWare.NUnitTestRunner.TestFixture.get_TearDown [method]: unit -> System.Reflection.MethodInfo list
 WoofWare.NUnitTestRunner.TestFixture.get_Tests [method]: unit -> WoofWare.NUnitTestRunner.SingleTestMethod list
+WoofWare.NUnitTestRunner.TestFixture.get_Type [method]: unit -> System.Type
 WoofWare.NUnitTestRunner.TestFixture.Name [property]: [read-only] string
 WoofWare.NUnitTestRunner.TestFixture.OneTimeSetUp [property]: [read-only] System.Reflection.MethodInfo option
 WoofWare.NUnitTestRunner.TestFixture.OneTimeTearDown [property]: [read-only] System.Reflection.MethodInfo option
+WoofWare.NUnitTestRunner.TestFixture.Parallelize [property]: [read-only] WoofWare.NUnitTestRunner.ClassParallelScope WoofWare.NUnitTestRunner.Parallelizable option
+WoofWare.NUnitTestRunner.TestFixture.Parameters [property]: [read-only] obj list list
 WoofWare.NUnitTestRunner.TestFixture.SetUp [property]: [read-only] System.Reflection.MethodInfo list
 WoofWare.NUnitTestRunner.TestFixture.TearDown [property]: [read-only] System.Reflection.MethodInfo list
 WoofWare.NUnitTestRunner.TestFixture.Tests [property]: [read-only] WoofWare.NUnitTestRunner.SingleTestMethod list
+WoofWare.NUnitTestRunner.TestFixture.Type [property]: [read-only] System.Type
 WoofWare.NUnitTestRunner.TestFixtureModule inherit obj
 WoofWare.NUnitTestRunner.TestFixtureModule.parse [static method]: System.Type -> WoofWare.NUnitTestRunner.TestFixture
-WoofWare.NUnitTestRunner.TestFixtureModule.run [static method]: WoofWare.NUnitTestRunner.ITestProgress -> (WoofWare.NUnitTestRunner.TestFixture -> WoofWare.NUnitTestRunner.SingleTestMethod -> bool) -> WoofWare.NUnitTestRunner.TestFixture -> WoofWare.NUnitTestRunner.FixtureRunResults
+WoofWare.NUnitTestRunner.TestFixtureModule.run [static method]: WoofWare.NUnitTestRunner.ITestProgress -> (WoofWare.NUnitTestRunner.TestFixture -> WoofWare.NUnitTestRunner.SingleTestMethod -> bool) -> WoofWare.NUnitTestRunner.TestFixture -> WoofWare.NUnitTestRunner.FixtureRunResults list
 WoofWare.NUnitTestRunner.TestKind inherit obj, implements WoofWare.NUnitTestRunner.TestKind System.IEquatable, System.Collections.IStructuralEquatable - union type with 3 cases
 WoofWare.NUnitTestRunner.TestKind+Data inherit WoofWare.NUnitTestRunner.TestKind
 WoofWare.NUnitTestRunner.TestKind+Data.get_Item [method]: unit -> obj list list
@@ -352,10 +416,12 @@ WoofWare.NUnitTestRunner.TrxOutcome.Parse [static method]: string -> WoofWare.NU
 WoofWare.NUnitTestRunner.TrxOutcome.Tag [property]: [read-only] int
 WoofWare.NUnitTestRunner.TrxOutcome.Warning [static property]: [read-only] WoofWare.NUnitTestRunner.TrxOutcome
 WoofWare.NUnitTestRunner.TrxOutput inherit obj, implements WoofWare.NUnitTestRunner.TrxOutput System.IEquatable, System.Collections.IStructuralEquatable
-WoofWare.NUnitTestRunner.TrxOutput..ctor [constructor]: (string option, WoofWare.NUnitTestRunner.TrxErrorInfo option)
+WoofWare.NUnitTestRunner.TrxOutput..ctor [constructor]: (string option, string option, WoofWare.NUnitTestRunner.TrxErrorInfo option)
 WoofWare.NUnitTestRunner.TrxOutput.ErrorInfo [property]: [read-only] WoofWare.NUnitTestRunner.TrxErrorInfo option
 WoofWare.NUnitTestRunner.TrxOutput.get_ErrorInfo [method]: unit -> WoofWare.NUnitTestRunner.TrxErrorInfo option
+WoofWare.NUnitTestRunner.TrxOutput.get_StdErr [method]: unit -> string option
 WoofWare.NUnitTestRunner.TrxOutput.get_StdOut [method]: unit -> string option
+WoofWare.NUnitTestRunner.TrxOutput.StdErr [property]: [read-only] string option
 WoofWare.NUnitTestRunner.TrxOutput.StdOut [property]: [read-only] string option
 WoofWare.NUnitTestRunner.TrxReport inherit obj, implements WoofWare.NUnitTestRunner.TrxReport System.IEquatable, System.Collections.IStructuralEquatable
 WoofWare.NUnitTestRunner.TrxReport..ctor [constructor]: (System.Guid, string, WoofWare.NUnitTestRunner.TrxReportTimes, WoofWare.NUnitTestRunner.TrxTestSettings, WoofWare.NUnitTestRunner.TrxUnitTestResult list, WoofWare.NUnitTestRunner.TrxUnitTest list, WoofWare.NUnitTestRunner.TrxTestEntry list, WoofWare.NUnitTestRunner.TrxTestListEntry list, WoofWare.NUnitTestRunner.TrxResultsSummary)

WoofWare.NUnitTestRunner.Lib/TestFixture.fs

@@ -1,6 +1,7 @@
 namespace WoofWare.NUnitTestRunner
 
 open System
+open System.Collections
 open System.Diagnostics
 open System.IO
 open System.Reflection
@@ -415,36 +416,15 @@ module TestFixture =
 
     /// Run every test (except those which fail the `filter`) in this test fixture, as well as the
     /// appropriate setup and tear-down logic.
-    let run
+    let private runOneFixture
         (progress : ITestProgress)
         (filter : TestFixture -> SingleTestMethod -> bool)
+        (name : string)
+        (containingObject : obj)
         (tests : TestFixture)
         : FixtureRunResults
         =
-        progress.OnTestFixtureStart tests.Name tests.Tests.Length
+        progress.OnTestFixtureStart name tests.Tests.Length
-
-        let containingObject =
-            let methods =
-                seq {
-                    match tests.OneTimeSetUp with
-                    | None -> ()
-                    | Some t -> yield t
-
-                    match tests.OneTimeTearDown with
-                    | None -> ()
-                    | Some t -> yield t
-
-                    yield! tests.Tests |> Seq.map (fun t -> t.Method)
-                }
-
-            methods
-            |> Seq.tryPick (fun mi ->
-                if not mi.IsStatic then
-                    Some (Activator.CreateInstance mi.DeclaringType)
-                else
-                    None
-            )
-            |> Option.toObj
 
         let oldWorkDir = Environment.CurrentDirectory
         Environment.CurrentDirectory <- FileInfo(tests.ContainingAssembly.Location).Directory.FullName
@@ -470,7 +450,7 @@ module TestFixture =
                 ExecutionId = Guid.NewGuid ()
                 TestId = Guid.NewGuid ()
                 // This one is a bit dubious, because we don't actually have a test name at all
-                TestName = tests.Name
+                TestName = name
                 ClassName = tests.Name
                 StdOut = if String.IsNullOrEmpty stdOut then None else Some stdOut
                 StdErr = if String.IsNullOrEmpty stdErr then None else Some stdErr
@@ -540,19 +520,52 @@ module TestFixture =
     /// Interpret this type as a [<TestFixture>], extracting the test members from it and annotating them with all
     /// relevant information about how we should run them.
     let parse (parentType : Type) : TestFixture =
-        if
+        let categories, args, par =
-            parentType.CustomAttributes
+            (([], [], None), parentType.CustomAttributes)
-            |> Seq.exists (fun attr -> attr.AttributeType.FullName = "NUnit.Framework.SetUpFixtureAttribute")
+            ||> Seq.fold (fun (categories, args, par) attr ->
-        then
+                match attr.AttributeType.FullName with
-            failwith "This test runner does not support SetUpFixture. Please shout if you want this."
+                | "NUnit.Framework.SetUpFixtureAttribute" ->
+                    failwith "This test runner does not support SetUpFixture. Please shout if you want this."
+                | "NUnit.Framework.CategoryAttribute" ->
+                    let cat = attr.ConstructorArguments |> Seq.exactlyOne |> _.Value |> unbox<string>
+                    cat :: categories, args, par
+                | "NUnit.Framework.TestFixtureAttribute" ->
+                    let newArgs =
+                        match attr.ConstructorArguments |> Seq.map _.Value |> Seq.toList with
+                        | [ :? ICollection as x ] ->
+                            x |> Seq.cast<CustomAttributeTypedArgument> |> Seq.map _.Value |> Seq.toList
+                        | xs -> xs
 
-        let categories =
+                    categories, newArgs :: args, par
-            parentType.CustomAttributes
+                | "NUnit.Framework.NonParallelizableAttribute" ->
-            |> Seq.filter (fun attr -> attr.AttributeType.FullName = "NUnit.Framework.CategoryAttribute")
+                    match par with
-            |> Seq.map (fun attr -> attr.ConstructorArguments |> Seq.exactlyOne |> _.Value |> unbox<string>)
+                    | Some _ -> failwith $"Got multiple parallelism attributes on %s{parentType.FullName}"
-            |> Seq.toList
+                    | None -> categories, args, Some Parallelizable.No
+                | "NUnit.Framework.ParallelizableAttribute" ->
+                    match par with
+                    | Some _ -> failwith $"Got multiple parallelism attributes on %s{parentType.FullName}"
+                    | None ->
+                        match attr.ConstructorArguments |> Seq.toList with
+                        | [] -> categories, args, Some (Parallelizable.Yes ClassParallelScope.Self)
+                        | [ v ] ->
+                            match v.Value with
+                            | :? int as v ->
+                                match v with
+                                | 512 -> categories, args, Some (Parallelizable.Yes ClassParallelScope.Fixtures)
+                                | 256 -> categories, args, Some (Parallelizable.Yes ClassParallelScope.Children)
+                                | 257 -> categories, args, Some (Parallelizable.Yes ClassParallelScope.All)
+                                | 1 -> categories, args, Some (Parallelizable.Yes ClassParallelScope.Self)
+                                | v ->
+                                    failwith
+                                        $"Could not recognise value %i{v} of parallel scope in %s{parentType.FullName}"
+                            | v ->
+                                failwith
+                                    $"Unexpectedly non-int value %O{v} of parallel scope in %s{parentType.FullName}"
+                        | _ -> failwith $"unexpectedly got multiple args to Parallelizable on %s{parentType.FullName}"
+                | _ -> categories, args, par
+            )
 
-        (TestFixture.Empty parentType.Assembly parentType.Name, parentType.GetRuntimeMethods ())
+        (TestFixture.Empty parentType par args, parentType.GetRuntimeMethods ())
         ||> Seq.fold (fun state mi ->
             ((state, []), mi.CustomAttributes)
             ||> Seq.fold (fun (state, unrecognisedAttrs) attr ->
@@ -622,3 +635,48 @@ module TestFixture =
 
                 state
         )
+
+    /// Run every test (except those which fail the `filter`) in this test fixture, as well as the
+    /// appropriate setup and tear-down logic.
+    let run
+        (progress : ITestProgress)
+        (filter : TestFixture -> SingleTestMethod -> bool)
+        (tests : TestFixture)
+        : FixtureRunResults list
+        =
+        match tests.Parameters with
+        | [] -> [ null ]
+        | args -> args |> List.map List.toArray
+        |> List.map (fun args ->
+            let containingObject =
+                let methods =
+                    seq {
+                        match tests.OneTimeSetUp with
+                        | None -> ()
+                        | Some t -> yield t
+
+                        match tests.OneTimeTearDown with
+                        | None -> ()
+                        | Some t -> yield t
+
+                        yield! tests.Tests |> Seq.map (fun t -> t.Method)
+                    }
+
+                methods
+                |> Seq.tryPick (fun mi ->
+                    if not mi.IsStatic then
+                        Some (Activator.CreateInstance (mi.DeclaringType, args))
+                    else
+                        None
+                )
+                |> Option.toObj
+
+            let name =
+                if isNull args then
+                    tests.Name
+                else
+                    let args = args |> Seq.map (fun o -> o.ToString ()) |> String.concat ","
+                    $"%s{tests.Name}(%s{args})"
+
+            runOneFixture progress filter name containingObject tests
+        )

WoofWare.NUnitTestRunner.Lib/TrxReport.fs

@@ -290,6 +290,8 @@ type TrxOutput =
     {
         /// What the entity printed to standard output.
         StdOut : string option
+        /// What the entity printed to standard error.
+        StdErr : string option
         /// Description of any error the entity encountered.
         ErrorInfo : TrxErrorInfo option
     }
@@ -305,6 +307,14 @@ type TrxOutput =
             childNode.AppendChild text |> ignore<XmlNode>
             node.AppendChild childNode |> ignore<XmlNode>
 
+        match this.StdErr with
+        | None -> ()
+        | Some stderr ->
+            let text = doc.CreateTextNode stderr
+            let childNode = doc.CreateElement ("StdErr", XmlUtil.NS)
+            childNode.AppendChild text |> ignore<XmlNode>
+            node.AppendChild childNode |> ignore<XmlNode>
+
         match this.ErrorInfo with
         | None -> ()
         | Some errInfo -> node.AppendChild (errInfo.toXml doc) |> ignore<XmlNode>
@@ -317,6 +327,11 @@ type TrxOutput =
             | NodeWithNamedChild "StdOut" (OneChildNode "StdOut" (NoChildrenNode stdout)) -> Some stdout
             | _ -> None
 
+        let stderr =
+            match node with
+            | NodeWithNamedChild "StdErr" (OneChildNode "StdErr" (NoChildrenNode stdout)) -> Some stdout
+            | _ -> None
+
         let errorInfo =
             match node with
             | NodeWithNamedChild "ErrorInfo" node ->
@@ -330,6 +345,7 @@ type TrxOutput =
         | Ok errorInfo ->
             {
                 StdOut = stdout
+                StdErr = stderr
                 ErrorInfo = errorInfo
             }
             |> Ok

WoofWare.NUnitTestRunner.Lib/WoofWare.NUnitTestRunner.Lib.fsproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-      <TargetFramework>netstandard2.1</TargetFramework>
+      <TargetFramework>net6.0</TargetFramework>
       <GenerateDocumentationFile>true</GenerateDocumentationFile>
       <Authors>Patrick Stevens</Authors>
       <Copyright>Copyright (c) Patrick Stevens 2024</Copyright>
@@ -14,10 +14,16 @@
       <PackageId>WoofWare.NUnitTestRunner.Lib</PackageId>
       <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
       <WarnOn>FS3559</WarnOn>
+      <WoofWareMyriadPluginVersion>2.1.44</WoofWareMyriadPluginVersion>
     </PropertyGroup>
 
     <ItemGroup>
       <Compile Include="AssemblyInfo.fs" />
+      <Compile Include="RuntimeConfig.fs" />
+      <Compile Include="GeneratedRuntimeConfig.fs">
+        <MyriadFile>RuntimeConfig.fs</MyriadFile>
+      </Compile>
+      <Compile Include="DotnetRuntime.fs" />
       <Compile Include="Array.fs" />
       <Compile Include="List.fs" />
       <Compile Include="Result.fs" />
@@ -38,6 +44,14 @@
   <ItemGroup>
     <PackageReference Include="WoofWare.PrattParser" Version="0.1.2" />
     <PackageReference Update="FSharp.Core" Version="6.0.0" />
+    <PackageReference Include="WoofWare.DotnetRuntimeLocator" Version="0.1.4" />
+    <PackageReference Include="WoofWare.Myriad.Plugins.Attributes" Version="3.1.6" />
+    <PackageReference Include="Myriad.SDK" Version="0.8.3" />
+    <PackageReference Include="WoofWare.Myriad.Plugins" Version="$(WoofWareMyriadPluginVersion)" PrivateAssets="all" />
+  </ItemGroup>
+  
+  <ItemGroup>
+    <MyriadSdkGenerator Include="$(NuGetPackageRoot)/woofware.myriad.plugins/$(WoofWareMyriadPluginVersion)/lib/net6.0/WoofWare.Myriad.Plugins.dll" />
   </ItemGroup>
 
 </Project>

WoofWare.NUnitTestRunner.Lib/version.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.8",
+  "version": "0.12",
   "publicReleaseRefSpec": [
     "^refs/heads/main$"
   ],

WoofWare.NUnitTestRunner/Program.fs

@@ -1,10 +1,10 @@
 namespace WoofWare.NUnitTestRunner
 
 open System
-open WoofWare.DotnetRuntimeLocator
 open System.IO
 open System.Reflection
 open System.Runtime.Loader
+open Spectre.Console
 
 // Fix for https://github.com/Smaug123/unofficial-nunit-runner/issues/8
 // Set AppContext.BaseDirectory to where the test DLL is.
@@ -41,101 +41,6 @@ type Ctx (dll : FileInfo, runtimes : DirectoryInfo list) =
 
 
 module Program =
-    let selectRuntime
-        (config : RuntimeOptions)
-        (f : DotnetEnvironmentInfo)
-        : Choice<DotnetEnvironmentFrameworkInfo, DotnetEnvironmentSdkInfo> option
-        =
-        let rollForward =
-            match Environment.GetEnvironmentVariable "DOTNET_ROLL_FORWARD" with
-            | null ->
-                config.RollForward
-                |> Option.map RollForward.Parse
-                |> Option.defaultValue RollForward.Minor
-            | s -> RollForward.Parse s
-
-        let desiredVersions =
-            match config.Framework with
-            | Some f -> [ Version f.Version, f.Name ]
-            | None ->
-
-            match config.Frameworks with
-            | Some f -> f |> List.map (fun f -> Version f.Version, f.Name)
-            | None ->
-                failwith
-                    "Could not deduce a framework version due to lack of either Framework or Frameworks in runtimeconfig"
-
-        let compatiblyNamedRuntimes =
-            f.Frameworks
-            |> Seq.collect (fun availableFramework ->
-                desiredVersions
-                |> List.choose (fun (desiredVersion, desiredName) ->
-                    if desiredName = availableFramework.Name then
-                        Some
-                            {|
-                                Desired = desiredVersion
-                                Name = desiredName
-                                Installed = availableFramework
-                                InstalledVersion = Version availableFramework.Version
-                            |}
-                    else
-                        None
-                )
-            )
-            |> Seq.toList
-
-        match rollForward with
-        | RollForward.Minor ->
-            let available =
-                compatiblyNamedRuntimes
-                |> Seq.filter (fun data ->
-                    data.InstalledVersion.Major = data.Desired.Major
-                    && data.InstalledVersion.Minor >= data.Desired.Minor
-                )
-                |> Seq.groupBy (fun data -> data.Name)
-                |> Seq.map (fun (name, data) ->
-                    let data =
-                        data
-                        |> Seq.minBy (fun data -> data.InstalledVersion.Minor, data.InstalledVersion.Build)
-
-                    name, data.Installed
-                )
-                // TODO: how do we select between many available frameworks?
-                |> Seq.tryHead
-
-            match available with
-            | Some (_, f) -> Some (Choice1Of2 f)
-            | None ->
-                // TODO: maybe we can ask the SDK. But we keep on trucking: maybe we're self-contained,
-                // and we'll actually find all the runtime next to the DLL.
-                None
-        | _ -> failwith "non-minor RollForward not supported yet; please shout if you want it"
-
-    let locateRuntimes (dll : FileInfo) : DirectoryInfo list =
-        let runtimeConfig =
-            let name =
-                if not (dll.Name.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) then
-                    failwith $"Expected DLL %s{dll.FullName} to end in .dll"
-
-                dll.Name.Substring (0, dll.Name.Length - 4)
-
-            Path.Combine (dll.Directory.FullName, $"%s{name}.runtimeconfig.json")
-            |> File.ReadAllText
-            |> System.Text.Json.Nodes.JsonNode.Parse
-            |> RuntimeConfig.jsonParse
-            |> fun f -> f.RuntimeOptions
-
-        let availableRuntimes = DotnetEnvironmentInfo.Get ()
-
-        let runtime = selectRuntime runtimeConfig availableRuntimes
-
-        match runtime with
-        | None ->
-            // Keep on trucking: let's be optimistic and hope that we're self-contained.
-            [ dll.Directory ]
-        | Some (Choice1Of2 runtime) -> [ dll.Directory ; DirectoryInfo $"%s{runtime.Path}/%s{runtime.Version}" ]
-        | Some (Choice2Of2 sdk) -> [ dll.Directory ; DirectoryInfo sdk.Path ]
-
     let main argv =
         let startTime = DateTimeOffset.Now
 
@@ -157,17 +62,61 @@ module Program =
             | Some filter -> Filter.shouldRun filter
             | None -> fun _ _ -> true
 
-        let progress = Progress.spectre ()
+        let stderr =
+            let consoleSettings = AnsiConsoleSettings ()
+            consoleSettings.Out <- AnsiConsoleOutput Console.Error
+            AnsiConsole.Create consoleSettings
+
+        let progress = Progress.spectre stderr
 
         use _ = new SetBaseDir (testDll)
 
-        let ctx = Ctx (testDll, locateRuntimes testDll)
+        let ctx = Ctx (testDll, DotnetRuntime.locate testDll)
         let assy = ctx.LoadFromAssemblyPath testDll.FullName
 
+        let levelOfParallelism, par =
+            ((None, None), assy.CustomAttributes)
+            ||> Seq.fold (fun (levelPar, par) attr ->
+                match attr.AttributeType.FullName with
+                | "NUnit.Framework.LevelOfParallelismAttribute" ->
+                    let arg = attr.ConstructorArguments |> Seq.exactlyOne |> _.Value |> unbox<int>
+
+                    match levelPar with
+                    | None -> (Some arg, par)
+                    | Some existing ->
+                        failwith $"Assembly %s{assy.Location} declares parallelism %i{arg} and also %i{existing}"
+                | "NUnit.Framework.NonParallelizableAttribute" ->
+                    match levelPar with
+                    | None -> (Some 1, par)
+                    | Some existing ->
+                        failwith
+                            $"Assembly %s{assy.Location} declares non-parallelizable and also parallelism %i{existing}"
+                | "NUnit.Framework.ParallelizableAttribute" ->
+                    match par with
+                    | Some _ -> failwith "Got multiple Parallelize attributes in assembly"
+                    | None ->
+                        match attr.ConstructorArguments |> Seq.toList with
+                        | [] -> levelPar, Some (Parallelizable.Yes AssemblyParallelScope.Fixtures)
+                        | [ v ] ->
+                            match v.Value with
+                            | :? int as v ->
+                                match v with
+                                | 512 -> levelPar, Some (Parallelizable.Yes AssemblyParallelScope.Fixtures)
+                                | 256 -> levelPar, Some (Parallelizable.Yes AssemblyParallelScope.Children)
+                                | 257 ->
+                                    failwith "ParallelScope.All is invalid on assemblies; only Fixtures or Children"
+                                | 1 ->
+                                    failwith "ParallelScope.Self is invalid on assemblies; only Fixtures or Children"
+                                | v -> failwith $"Could not recognise value %i{v} of parallel scope on assembly"
+                            | v -> failwith $"Unexpectedly non-int value %O{v} of parallel scope on assembly"
+                        | _ -> failwith "unexpectedly got multiple args to Parallelizable on assembly"
+                | _ -> levelPar, par
+            )
+
         let testFixtures = assy.ExportedTypes |> Seq.map TestFixture.parse |> Seq.toList
 
         let creationTime = DateTimeOffset.Now
-        let results = testFixtures |> List.map (TestFixture.run progress filter)
+        let results = testFixtures |> List.collect (TestFixture.run progress filter)
 
         let finishTime = DateTimeOffset.Now
         let finishTimeHumanReadable = finishTime.ToString @"yyyy-MM-dd HH:mm:ss"
@@ -270,6 +219,7 @@ module Program =
                 Output =
                     {
                         StdOut = None
+                        StdErr = None
                         ErrorInfo = None
                     }
                 RunInfos =
@@ -373,11 +323,11 @@ module Program =
                     Output =
                         match i.StdOut, i.StdErr, exc with
                         | None, None, None -> None
-                        // TODO surely stderr can be emitted
+                        | stdout, stderr, exc ->
-                        | stdout, _stderr, exc ->
                             Some
                                 {
                                     TrxOutput.StdOut = stdout
+                                    StdErr = stderr
                                     ErrorInfo = exc
                                 }
                 }

WoofWare.NUnitTestRunner/Progress.fs

@@ -4,21 +4,21 @@ open Spectre.Console
 
 [<RequireQualifiedAccess>]
 module Progress =
-    let spectre () : ITestProgress =
+    let spectre (console : IAnsiConsole) : ITestProgress =
         { new ITestProgress with
             member _.OnTestFailed name failure =
-                AnsiConsole.Console.MarkupLine
+                console.MarkupLine
                     $"[red]Test '%s{Markup.Escape name}' failed: %s{Markup.Escape (failure.ToString ())}[/]"
 
             member _.OnTestFixtureStart name testCount =
-                AnsiConsole.Console.MarkupLine $"[white]Running tests: %s{Markup.Escape name}[/]"
+                console.MarkupLine $"[white]Running tests: %s{Markup.Escape name}[/]"
 
             member _.OnTestMemberFinished name =
-                AnsiConsole.Console.MarkupLine $"[gray]Finished test: %s{Markup.Escape name}[/]"
+                console.MarkupLine $"[gray]Finished test: %s{Markup.Escape name}[/]"
 
             member _.OnTestMemberSkipped name =
-                AnsiConsole.Console.MarkupLine $"[yellow]Skipping test due to filter: %s{Markup.Escape name}[/]"
+                console.MarkupLine $"[yellow]Skipping test due to filter: %s{Markup.Escape name}[/]"
 
             member _.OnTestMemberStart name =
-                AnsiConsole.Console.MarkupLine $"[white]Running test: %s{Markup.Escape name}[/]"
+                console.MarkupLine $"[white]Running test: %s{Markup.Escape name}[/]"
         }

WoofWare.NUnitTestRunner/WoofWare.NUnitTestRunner.Test/TestList.fs

@@ -11,6 +11,8 @@ module TestList =
     [<Test>]
     let ``combinations has right size`` () =
         let property (xs : int list list) =
+            let xs = if xs.Length > 6 then xs |> List.take 6 else xs
+            let xs = xs |> List.map (fun xs -> if xs.Length > 6 then xs |> List.take 6 else xs)
             let combs = List.combinations xs
 
             combs.Length

WoofWare.NUnitTestRunner/WoofWare.NUnitTestRunner.Test/TestTrx.fs

@@ -197,6 +197,7 @@ Running all tests in /Users/patrick/Documents/GitHub/TestRunner/TestRunner/TestR
 Ensure version is monotonic: Not yet published
 NUnit Adapter 4.5.0.0: Test execution complete
 """
+                        StdErr = None
                         ErrorInfo = None
                     }
                 Outcome = TrxOutcome.Failed

WoofWare.NUnitTestRunner/WoofWare.NUnitTestRunner.fsproj

@@ -16,14 +16,9 @@
     <PackageId>WoofWare.NUnitTestRunner</PackageId>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     <WarnOn>FS3559</WarnOn>
-    <WoofWareMyriadPluginVersion>2.1.42</WoofWareMyriadPluginVersion>
   </PropertyGroup>
 
   <ItemGroup>
-    <Compile Include="RuntimeConfig.fs" />
-    <Compile Include="GeneratedRuntimeConfig.fs">
-      <MyriadFile>RuntimeConfig.fs</MyriadFile>
-    </Compile>
     <Compile Include="Seq.fs" />
     <Compile Include="Progress.fs" />
     <Compile Include="Program.fs" />
@@ -39,14 +34,6 @@
 
   <ItemGroup>
     <PackageReference Include="Spectre.Console" Version="0.49.1" />
-    <PackageReference Include="WoofWare.DotnetRuntimeLocator" Version="0.1.4" />
-    <PackageReference Include="WoofWare.Myriad.Plugins.Attributes" Version="3.1.6" />
-    <PackageReference Include="Myriad.SDK" Version="0.8.3" />
-    <PackageReference Include="WoofWare.Myriad.Plugins" Version="$(WoofWareMyriadPluginVersion)" PrivateAssets="all" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <MyriadSdkGenerator Include="$(NuGetPackageRoot)/woofware.myriad.plugins/$(WoofWareMyriadPluginVersion)/lib/net6.0/WoofWare.Myriad.Plugins.dll" />
   </ItemGroup>
 
   <ItemGroup>

WoofWare.NUnitTestRunner/version.json

@@ -5,6 +5,7 @@
   ],
   "pathFilters": [
     "./",
+    "^./WoofWare.NUnitTestRunner.Test",
     ":/WoofWare.NUnitTestRunner.Lib",
     ":/Directory.Build.props",
     ":/README.md"

nix/deps.nix

@@ -36,6 +36,31 @@
     version = "6.0.0";
     sha256 = "18q3p0z155znwj1l0qq3vq9nh9wl2i4mlfx4pmrnia4czr0xdkmb";
   })
+  (fetchNuGet {
+    pname = "Microsoft.AspNetCore.App.Ref";
+    version = "6.0.30";
+    sha256 = "17k3l8xd5bsyk69bm5q4nxbpb4i0izw1kzmzi7j3p8pmm9prgrpy";
+  })
+  (fetchNuGet {
+    pname = "Microsoft.AspNetCore.App.Runtime.linux-arm64";
+    version = "6.0.30";
+    sha256 = "1n4v5przbrjhzj01b6qijpdc2jbsxr66ijvd0483qxh4s0b4jppr";
+  })
+  (fetchNuGet {
+    pname = "Microsoft.AspNetCore.App.Runtime.linux-x64";
+    version = "6.0.30";
+    sha256 = "18v0l07q74m5xxaf6y6dkmr6and8ivya0nslffnr4djrxcbiygdr";
+  })
+  (fetchNuGet {
+    pname = "Microsoft.AspNetCore.App.Runtime.osx-arm64";
+    version = "6.0.30";
+    sha256 = "0p53lyqmr5n2ym202pbgmsd9b9aa6jar7ic04dcq86h2g77r5jqk";
+  })
+  (fetchNuGet {
+    pname = "Microsoft.AspNetCore.App.Runtime.osx-x64";
+    version = "6.0.30";
+    sha256 = "009srl8vazkjnd93xr6k1m353spbki9gn1yzp4zgazgbrini6rqc";
+  })
   (fetchNuGet {
     pname = "Microsoft.CodeCoverage";
     version = "17.10.0";
@@ -46,6 +71,51 @@
     version = "17.10.0";
     sha256 = "13g8fwl09li8fc71nk13dgkb7gahd4qhamyg2xby7am63nlchhdf";
   })
+  (fetchNuGet {
+    pname = "Microsoft.NETCore.App.Host.linux-arm64";
+    version = "6.0.30";
+    sha256 = "0l3gjhmnjd5n67w83smqyhmfcwzszafjgcbq8kdwxiwwh2m6nh2i";
+  })
+  (fetchNuGet {
+    pname = "Microsoft.NETCore.App.Host.linux-x64";
+    version = "6.0.30";
+    sha256 = "0ss3108c2h7afypvliyqixv4dll60sq9iwqy90k1p132znpszrmb";
+  })
+  (fetchNuGet {
+    pname = "Microsoft.NETCore.App.Host.osx-arm64";
+    version = "6.0.30";
+    sha256 = "08k5v35mvcs712kb0vcfjd1dsas5rgwrmv8rn87mzjb2p6ajl3n3";
+  })
+  (fetchNuGet {
+    pname = "Microsoft.NETCore.App.Host.osx-x64";
+    version = "6.0.30";
+    sha256 = "02x38c68xan8hlr59mindcl4rcx49bbh4bibh6fw1l4rrijb38lw";
+  })
+  (fetchNuGet {
+    pname = "Microsoft.NETCore.App.Ref";
+    version = "6.0.30";
+    sha256 = "1wqqjhhlqz4dmijcx3kg3hnwq0s0jzqsddaksskzhq8avr4ziy18";
+  })
+  (fetchNuGet {
+    pname = "Microsoft.NETCore.App.Runtime.linux-arm64";
+    version = "6.0.30";
+    sha256 = "0xfhcig3gj3986rxp3dhnd8hvnj4nvyhz1fz7kpx342d3g53wb37";
+  })
+  (fetchNuGet {
+    pname = "Microsoft.NETCore.App.Runtime.linux-x64";
+    version = "6.0.30";
+    sha256 = "1s81sj8lnb8szqawxh3vc8wi815ln12cyhrl3f7hwcpay57xgswx";
+  })
+  (fetchNuGet {
+    pname = "Microsoft.NETCore.App.Runtime.osx-arm64";
+    version = "6.0.30";
+    sha256 = "0s71k92daakzwish65gmn4nniy6bf2hv34c0sb6m1hv3criqxmp4";
+  })
+  (fetchNuGet {
+    pname = "Microsoft.NETCore.App.Runtime.osx-x64";
+    version = "6.0.30";
+    sha256 = "0xybqg2wd240r1nm2vrbn2qbfqfnqsmxn1012zzwjn17wa2si9a1";
+  })
   (fetchNuGet {
     pname = "Microsoft.NETCore.Platforms";
     version = "2.0.0";
@@ -71,11 +141,6 @@
     version = "3.6.139";
     sha256 = "0npcryhq3r0c2zi940jk39h13mzc4hyg7z8gm6jdmxi1aqv1vh8c";
   })
-  (fetchNuGet {
-    pname = "NETStandard.Library.Ref";
-    version = "2.1.0";
-    sha256 = "12n76gymxq715lkrw841vi5r84kx746cxxssp22pd08as75jzsj6";
-  })
   (fetchNuGet {
     pname = "Newtonsoft.Json";
     version = "13.0.1";
@@ -188,8 +253,8 @@
   })
   (fetchNuGet {
     pname = "WoofWare.Myriad.Plugins";
-    version = "2.1.42";
+    version = "2.1.44";
-    sha256 = "0px46m734gsn1xa97111v1nwkyc2j52bw7z4bjdljzkmzzmnqa91";
+    sha256 = "0rp9hpkah60gd9x0ba2izr9ff1g7yhzv5a4pkhi5fbrwf5rpqpwx";
   })
   (fetchNuGet {
     pname = "WoofWare.Myriad.Plugins.Attributes";