Delete unused file (#94)

.editorconfig

@@ -1,40 +1,40 @@
-root=true
+root = true
 
 [*]
-charset=utf-8
-trim_trailing_whitespace=true
-insert_final_newline=true
-indent_style=space
-indent_size=4
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+indent_style = space
+indent_size = 4
 
 # ReSharper properties
-resharper_xml_indent_size=2
-resharper_xml_max_line_length=100
-resharper_xml_tab_width=2
+resharper_xml_indent_size = 2
+resharper_xml_max_line_length = 100
+resharper_xml_tab_width = 2
 
 [*.{csproj,fsproj,sqlproj,targets,props,ts,tsx,css,json}]
-indent_style=space
-indent_size=2
+indent_style = space
+indent_size = 2
 
 [*.{fs,fsi}]
-fsharp_bar_before_discriminated_union_declaration=true
-fsharp_space_before_uppercase_invocation=true
-fsharp_space_before_class_constructor=true
-fsharp_space_before_member=true
-fsharp_space_before_colon=true
-fsharp_space_before_semicolon=true
-fsharp_multiline_bracket_style=aligned
-fsharp_newline_between_type_definition_and_members=true
-fsharp_align_function_signature_to_indentation=true
-fsharp_alternative_long_member_definitions=true
-fsharp_multi_line_lambda_closing_newline=true
-fsharp_experimental_keep_indent_in_branch=true
-fsharp_max_value_binding_width=80
-fsharp_max_record_width=0
-max_line_length=120
-end_of_line=lf
+fsharp_bar_before_discriminated_union_declaration = true
+fsharp_space_before_uppercase_invocation = true
+fsharp_space_before_class_constructor = true
+fsharp_space_before_member = true
+fsharp_space_before_colon = true
+fsharp_space_before_semicolon = true
+fsharp_multiline_bracket_style = aligned
+fsharp_newline_between_type_definition_and_members = true
+fsharp_align_function_signature_to_indentation = true
+fsharp_alternative_long_member_definitions = true
+fsharp_multi_line_lambda_closing_newline = true
+fsharp_experimental_keep_indent_in_branch = true
+fsharp_max_value_binding_width = 80
+fsharp_max_record_width = 0
+max_line_length = 120
+end_of_line = lf
 
 [*.{appxmanifest,build,dtd,nuspec,xaml,xamlx,xoml,xsd}]
-indent_style=space
-indent_size=2
-tab_width=2
+indent_style = space
+indent_size = 2
+tab_width = 2

.envrc

@@ -0,0 +1 @@
+use flake

.github/workflows/assert-contents.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+echo "Unzipping version from NuGet"
+ls from-nuget.nupkg
+mkdir from-nuget && cp from-nuget.nupkg from-nuget/zip.zip && cd from-nuget && unzip zip.zip && rm zip.zip && cd - || exit 1
+
+echo "Unzipping version from local build"
+ls packed/
+mkdir from-local && cp packed/*.nupkg from-local/zip.zip && cd from-local && unzip zip.zip && rm zip.zip && cd - || exit 1
+
+cd from-local && find . -type f -exec sha256sum {} \; | sort > ../from-local.txt && cd .. || exit 1
+cd from-nuget && find . -type f -and -not -name '.signature.p7s' -exec sha256sum {} \; | sort > ../from-nuget.txt && cd .. || exit 1
+
+diff from-local.txt from-nuget.txt

.github/workflows/dotnet.yaml

@@ -57,6 +57,10 @@ jobs:
       security-events: none
       statuses: read
 
+    strategy:
+      matrix:
+        framework: ["net6.0", "net8.0"]
+
     steps:
     - uses: actions/checkout@v4
       with:
@@ -71,7 +75,7 @@ jobs:
     - name: Build
       run: 'nix develop --command dotnet build --no-restore --configuration Release'
     - name: Test using self
-      run: 'nix develop --command dotnet exec ./WoofWare.NUnitTestRunner/bin/Release/net8.0/WoofWare.NUnitTestRunner.dll ./Consumer/bin/Release/net8.0/Consumer.dll --trx TrxOut/out.trx'
+      run: 'nix develop --command dotnet exec ./WoofWare.NUnitTestRunner/bin/Release/net8.0/WoofWare.NUnitTestRunner.dll ./Consumer/bin/Release/${{matrix.framework}}/Consumer.dll --trx TrxOut/out.trx'
     - name: Parse Trx files
       uses: NasAmin/trx-parser@v0.6.0
       if: always()
@@ -240,11 +244,53 @@ jobs:
     steps:
       - run: echo "All required checks complete."
 
-  nuget-publish:
+  attestation-lib:
+    runs-on: ubuntu-latest
+    needs: [all-required-checks-complete]
+    if: ${{ !github.event.repository.fork && github.ref == 'refs/heads/main' }}
+    permissions:
+      id-token: write
+      attestations: write
+      contents: read
+    steps:
+      - name: Download NuGet artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: nuget-package-lib
+          path: packed
+      - name: Attest Build Provenance
+        uses: actions/attest-build-provenance@897ed5eab6ed058a474202017ada7f40bfa52940 # v1.0.0
+        with:
+          subject-path: "packed/*.nupkg"
+
+  attestation-tool:
+    runs-on: ubuntu-latest
+    needs: [all-required-checks-complete]
+    if: ${{ !github.event.repository.fork && github.ref == 'refs/heads/main' }}
+    permissions:
+      id-token: write
+      attestations: write
+      contents: read
+    steps:
+      - name: Download NuGet artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: nuget-package-tool
+          path: packed
+      - name: Attest Build Provenance
+        uses: actions/attest-build-provenance@897ed5eab6ed058a474202017ada7f40bfa52940 # v1.0.0
+        with:
+          subject-path: "packed/*.nupkg"
+
+  nuget-publish-lib:
     runs-on: ubuntu-latest
     if: ${{ !github.event.repository.fork && github.ref == 'refs/heads/main' }}
     needs: [all-required-checks-complete]
     environment: main-deploy
+    permissions:
+      id-token: write
+      attestations: write
+      contents: read
     steps:
       - uses: actions/checkout@v4
       - name: Install Nix
@@ -252,20 +298,73 @@ jobs:
         with:
           extra_nix_config: |
             access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
-      - name: Download NuGet artifact (lib)
+      - name: Download NuGet artifact
         uses: actions/download-artifact@v4
         with:
           name: nuget-package-lib
-          path: packed-lib
-      - name: Publish to NuGet (lib)
-        run: nix develop --command dotnet nuget push "packed-lib/WoofWare.NUnitTestRunner.Lib.*.nupkg" --api-key ${{ secrets.NUGET_API_KEY }} --source https://api.nuget.org/v3/index.json --skip-duplicate
-      - name: Download NuGet artifact (tool)
+          path: packed
+      - name: Publish to NuGet
+        id: publish-success
+        env:
+          NUGET_API_KEY: ${{ secrets.NUGET_API_KEY }}
+        run: 'nix develop --command bash ./.github/workflows/nuget-push.sh "packed/WoofWare.NUnitTestRunner.Lib.*.nupkg"'
+      - name: Wait for availability
+        if: steps.publish-success.outputs.result == 'published'
+        env:
+          PACKAGE_VERSION: ${{ steps.publish-success.outputs.version }}
+        run: 'echo "$PACKAGE_VERSION" && while ! curl -L --fail -o from-nuget.nupkg "https://www.nuget.org/api/v2/package/WoofWare.NUnitTestRunner.Lib/$PACKAGE_VERSION" ; do sleep 10; done'
+      # Astonishingly, NuGet.org considers it to be "more secure" to tamper with my package after upload (https://devblogs.microsoft.com/nuget/introducing-repository-signatures/).
+      # So we have to *re-attest* it after it's uploaded. Mind-blowing.
+      - name: Assert package contents
+        if: steps.publish-success.outputs.result == 'published'
+        run: 'bash ./.github/workflows/assert-contents.sh'
+      - name: Attest Build Provenance
+        if: steps.publish-success.outputs.result == 'published'
+        uses: actions/attest-build-provenance@897ed5eab6ed058a474202017ada7f40bfa52940 # v1.0.0
+        with:
+          subject-path: "from-nuget.nupkg"
+
+  nuget-publish-tool:
+    runs-on: ubuntu-latest
+    if: ${{ !github.event.repository.fork && github.ref == 'refs/heads/main' }}
+    needs: [all-required-checks-complete]
+    environment: main-deploy
+    permissions:
+      id-token: write
+      attestations: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Nix
+        uses: cachix/install-nix-action@V27
+        with:
+          extra_nix_config: |
+            access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
+      - name: Download NuGet artifact
         uses: actions/download-artifact@v4
         with:
           name: nuget-package-tool
-          path: packed-tool
-      - name: Publish to NuGet (tool)
-        run: nix develop --command dotnet nuget push "packed-tool/WoofWare.NUnitTestRunner.*.nupkg" --api-key ${{ secrets.NUGET_API_KEY }} --source https://api.nuget.org/v3/index.json --skip-duplicate
+          path: packed
+      - name: Publish to NuGet
+        id: publish-success
+        env:
+          NUGET_API_KEY: ${{ secrets.NUGET_API_KEY }}
+        run: 'nix develop --command bash ./.github/workflows/nuget-push.sh "packed/WoofWare.NUnitTestRunner.*.nupkg"'
+      - name: Wait for availability
+        if: steps.publish-success.outputs.result == 'published'
+        env:
+          PACKAGE_VERSION: ${{ steps.publish-success.outputs.version }}
+        run: 'echo "$PACKAGE_VERSION" && while ! curl -L --fail -o from-nuget.nupkg "https://www.nuget.org/api/v2/package/WoofWare.NUnitTestRunner/$PACKAGE_VERSION" ; do sleep 10; done'
+      # Astonishingly, NuGet.org considers it to be "more secure" to tamper with my package after upload (https://devblogs.microsoft.com/nuget/introducing-repository-signatures/).
+      # So we have to *re-attest* it after it's uploaded. Mind-blowing.
+      - name: Assert package contents
+        if: steps.publish-success.outputs.result == 'published'
+        run: 'bash ./.github/workflows/assert-contents.sh'
+      - name: Attest Build Provenance
+        if: steps.publish-success.outputs.result == 'published'
+        uses: actions/attest-build-provenance@897ed5eab6ed058a474202017ada7f40bfa52940 # v1.0.0
+        with:
+          subject-path: "from-nuget.nupkg"
 
   github-release-tool:
     runs-on: ubuntu-latest

.github/workflows/nuget-push.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+SOURCE_NUPKG=$(find . -type f -name '*.nupkg')
+
+PACKAGE_VERSION=$(basename "$SOURCE_NUPKG" | rev | cut -d '.' -f 2-4 | rev)
+
+echo "version=$PACKAGE_VERSION" >> "$GITHUB_OUTPUT"
+
+tmp=$(mktemp)
+
+if ! dotnet nuget push "$SOURCE_NUPKG" --api-key "$NUGET_API_KEY" --source https://api.nuget.org/v3/index.json > "$tmp" ; then
+    cat "$tmp"
+    if grep 'already exists and cannot be modified' "$tmp" ; then
+        echo "result=skipped" >> "$GITHUB_OUTPUT"
+        exit 0
+    else
+        echo "Unexpected failure to upload"
+        exit 1
+    fi
+fi
+
+cat "$tmp"
+
+echo "result=published" >> "$GITHUB_OUTPUT"

Consumer/Consumer.fsproj

@@ -1,8 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFramework>net8.0</TargetFramework>
-
+        <TargetFrameworks>net6.0;net8.0</TargetFrameworks>
         <IsPackable>false</IsPackable>
         <IsTestProject>true</IsTestProject>
     </PropertyGroup>
@@ -10,7 +9,9 @@
     <ItemGroup>
         <Compile Include="NoAttribute.fs" />
         <Compile Include="Inconclusive.fs" />
+        <Compile Include="TestNonParallel.fs" />
         <Compile Include="TestParallel.fs" />
+        <Compile Include="TestStdout.fs" />
         <Compile Include="TestParameterisedFixture.fs" />
         <Compile Include="TestSetUp.fs" />
         <Compile Include="TestValues.fs" />
@@ -24,9 +25,9 @@
 
     <ItemGroup>
       <PackageReference Include="FsUnit" Version="6.0.0" />
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.10.0"/>
-        <PackageReference Include="NUnit" Version="4.1.0"/>
-        <PackageReference Include="NUnit3TestAdapter" Version="4.5.0"/>
+      <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.10.0"/>
+      <PackageReference Include="NUnit" Version="4.1.0"/>
+      <PackageReference Include="NUnit3TestAdapter" Version="4.5.0"/>
     </ItemGroup>
 
 </Project>

Consumer/TestNonParallel.fs

@@ -0,0 +1,19 @@
+namespace Consumer
+
+open System
+open System.Threading
+open NUnit.Framework
+open FsUnitTyped
+
+[<TestFixture>]
+[<NonParallelizable>]
+module TestNonParallel =
+    let defaults = List.init 40 id
+    let lock = ref 0
+
+    [<TestCaseSource(nameof defaults)>]
+    let ``Default thing, but not parallel`` (i : int) =
+        Interlocked.Increment lock |> shouldEqual 1
+        Thread.Sleep (TimeSpan.FromMilliseconds (float i))
+        lock.Value <- 0
+        i |> shouldEqual i

Consumer/TestParallel.fs

@@ -1,5 +1,7 @@
 namespace Consumer
 
+open System
+open System.Threading
 open NUnit.Framework
 open FsUnitTyped
 
@@ -7,43 +9,58 @@ open FsUnitTyped
 [<Parallelizable>]
 module TestParallelDefault =
 
-    let defaults = List.init 100 id
+    let defaults = List.init 60 id
 
     [<TestCaseSource(nameof defaults)>]
-    let ``Default thing`` (i : int) = i |> shouldEqual i
+    let ``Default thing, no scope`` (i : int) =
+        Console.WriteLine i
+        Thread.Sleep (TimeSpan.FromMilliseconds (float i))
+        i |> shouldEqual i
 
 [<TestFixture>]
 [<Parallelizable(ParallelScope.All)>]
 module TestParallelAllScope =
 
-    let defaults = List.init 100 id
+    let defaults = List.init 60 id
 
     [<TestCaseSource(nameof defaults)>]
-    let ``Default thing`` (i : int) = i |> shouldEqual i
+    let ``Thing, all scope`` (i : int) =
+        Console.WriteLine i
+        Thread.Sleep (TimeSpan.FromMilliseconds (float i))
+        i |> shouldEqual i
 
 [<TestFixture>]
 [<Parallelizable(ParallelScope.Self)>]
 module TestParallelSelfScope =
 
-    let defaults = List.init 100 id
+    let defaults = List.init 60 id
 
     [<TestCaseSource(nameof defaults)>]
-    let ``Default thing`` (i : int) = i |> shouldEqual i
+    let ``Thing, self scope`` (i : int) =
+        Console.WriteLine i
+        Thread.Sleep (TimeSpan.FromMilliseconds (float i))
+        i |> shouldEqual i
 
 [<TestFixture>]
 [<Parallelizable(ParallelScope.Children)>]
 module TestParallelChildrenScope =
 
-    let defaults = List.init 100 id
+    let defaults = List.init 60 id
 
     [<TestCaseSource(nameof defaults)>]
-    let ``Default thing`` (i : int) = i |> shouldEqual i
+    let ``Thing, children scope`` (i : int) =
+        Console.WriteLine i
+        Thread.Sleep (TimeSpan.FromMilliseconds (float i))
+        i |> shouldEqual i
 
 [<TestFixture>]
 [<Parallelizable(ParallelScope.Fixtures)>]
 module TestParallelFixturesScope =
 
-    let defaults = List.init 100 id
+    let defaults = List.init 60 id
 
     [<TestCaseSource(nameof defaults)>]
-    let ``Default thing`` (i : int) = i |> shouldEqual i
+    let ``Thing, fixtures scope`` (i : int) =
+        Console.WriteLine i
+        Thread.Sleep (TimeSpan.FromMilliseconds (float i))
+        i |> shouldEqual i

Consumer/TestSetUp.fs

@@ -1,5 +1,6 @@
 namespace Consumer
 
+open System
 open FsUnitTyped
 open System.Threading
 open NUnit.Framework
@@ -11,6 +12,8 @@ module TestSetUp =
 
     [<OneTimeSetUp>]
     let oneTimeSetUp () =
+        Console.WriteLine "I'm being set up for the first time!"
+
         if Interlocked.Increment haveOneTimeSetUp <> 1 then
             failwith "one time setup happened more than once"
 
@@ -22,12 +25,14 @@ module TestSetUp =
 
     [<SetUp>]
     let setUp () =
+        Console.WriteLine "It's a set-up!"
         haveOneTimeSetUp.Value |> shouldEqual 1
         let newId = Interlocked.Increment setUpTimes
         lock setUpTimesSeen (fun () -> setUpTimesSeen.Add newId)
 
     [<TearDown>]
     let tearDown () =
+        Console.WriteLine "I'm a tear-down!"
         let newId = Interlocked.Increment tearDownTimes
         lock tearDownTimesSeen (fun () -> tearDownTimesSeen.Add newId)
 
@@ -35,19 +40,23 @@ module TestSetUp =
 
     [<OneTimeTearDown>]
     let oneTimeTearDown () =
+        Console.WriteLine "I'm being torn down, finally!"
+
         if Interlocked.Increment haveOneTimeTearDown <> 1 then
             failwith "one time tear down happened more than once"
 
         setUpTimesSeen
         |> Seq.toList
+        |> List.sort
         // Six tests: one for Test, two for the TestCase, three for the Repeat.
         |> shouldEqual [ 1..6 ]
 
-        tearDownTimesSeen |> Seq.toList |> shouldEqual [ 1..6 ]
+        tearDownTimesSeen |> Seq.toList |> List.sort |> shouldEqual [ 1..6 ]
 
     [<Test>]
     let ``Test 1`` () =
         haveOneTimeTearDown.Value |> shouldEqual 0
+        Console.WriteLine "By the way, I'm test 1"
         1 |> shouldEqual 1
 
     [<TestCase "h">]

Consumer/TestStdout.fs

@@ -0,0 +1,13 @@
+namespace Consumer
+
+open System
+open NUnit.Framework
+
+[<TestFixture>]
+module TestStdout =
+
+    [<Test>]
+    let ``Stdout is redirected`` () =
+        Console.Out.WriteLine "Hi!"
+        Console.WriteLine "Hi! part 2"
+        Console.Error.WriteLine "Bye!"

Consumer/TestValues.fs

@@ -86,14 +86,42 @@ module TestValues =
 
     [<OneTimeTearDown>]
     let ``Values are all OK`` () =
-        seen1 |> Seq.toList |> shouldEqual [ true ; false ]
-        seen2 |> Seq.toList |> shouldEqual [ (true, false) ; (false, true) ]
-        seen3 |> Seq.toList |> shouldEqual [ (88, box 29) ; (31, box 0) ]
-        seen4 |> Seq.toList |> shouldEqual [ ("hi", box "ohh") ; ("bye", null) ]
-        seen5 |> Seq.toList |> shouldEqual [ (88, box 29) ; (31, box 29) ]
-        seen6 |> Seq.toList |> shouldEqual [ ("hi", box "ohh") ; ("bye", box "ohh") ]
-        seen7 |> Seq.toList |> shouldEqual [ (88, box 29) ; (31, box 29) ]
-        seen8 |> Seq.toList |> shouldEqual [ ("hi", box "ohh") ; ("bye", box "ohh") ]
+        seen1 |> Seq.toList |> List.sort |> shouldEqual [ false ; true ]
+
+        seen2
+        |> Seq.toList
+        |> List.sort
+        |> shouldEqual [ (false, true) ; (true, false) ]
+
+        seen3
+        |> Seq.toList
+        |> List.sortBy fst
+        |> shouldEqual [ (31, box 0) ; (88, box 29) ]
+
+        seen4
+        |> Seq.toList
+        |> List.sortBy fst
+        |> shouldEqual [ ("bye", null) ; ("hi", box "ohh") ]
+
+        seen5
+        |> Seq.toList
+        |> List.sortBy fst
+        |> shouldEqual [ (31, box 29) ; (88, box 29) ]
+
+        seen6
+        |> Seq.toList
+        |> List.sortBy fst
+        |> shouldEqual [ ("bye", box "ohh") ; ("hi", box "ohh") ]
+
+        seen7
+        |> Seq.toList
+        |> List.sortBy fst
+        |> shouldEqual [ (31, box 29) ; (88, box 29) ]
+
+        seen8
+        |> Seq.toList
+        |> List.sortBy fst
+        |> shouldEqual [ ("bye", box "ohh") ; ("hi", box "ohh") ]
 
         seen9
         |> Seq.toList

WoofWare.NUnitTestRunner.Lib/AppContext.fs

@@ -0,0 +1,34 @@
+namespace WoofWare.NUnitTestRunner
+
+open System
+open System.IO
+open System.Reflection
+
+// Fix for https://github.com/Smaug123/unofficial-nunit-runner/issues/8
+// (This tells the DLL loader to look next to the test DLL for dependencies.)
+// TODO: do we actually need this when we move to the Hooks world?
+/// Context manager to set the AppContext.BaseDirectory of the executing DLL.
+type SetBaseDir (testDll : FileInfo) =
+    let oldBaseDir = AppContext.BaseDirectory
+
+    let setData =
+        let appContext = Type.GetType "System.AppContext"
+
+        if Object.ReferenceEquals (appContext, (null : obj)) then
+            ignore<string * string>
+        else
+
+        let setDataMethod =
+            appContext.GetMethod ("SetData", BindingFlags.Static ||| BindingFlags.Public)
+
+        if Object.ReferenceEquals (setDataMethod, (null : obj)) then
+            ignore<string * string>
+        else
+
+        fun (k, v) -> setDataMethod.Invoke ((null : obj), [| k ; v |]) |> unbox<unit>
+
+    do setData ("APP_CONTEXT_BASE_DIRECTORY", testDll.Directory.FullName)
+
+    interface IDisposable with
+        member _.Dispose () =
+            setData ("APP_CONTEXT_BASE_DIRECTORY", oldBaseDir)

WoofWare.NUnitTestRunner.Lib/Args.fs

@@ -0,0 +1,99 @@
+namespace WoofWare.NUnitTestRunner
+
+open System
+open System.IO
+
+[<AutoOpen>]
+module internal Patterns =
+    let (|Key|_|) (start : string) (s : string) : string option =
+        if s.StartsWith (start + "=", StringComparison.Ordinal) then
+            s.Substring (start.Length + 1) |> Some
+        else
+            None
+
+/// Represents how verbose the test runner's logging should be.
+[<RequireQualifiedAccess>]
+type LogLevel =
+    /// Don't log any information about the test run.
+    | Nothing
+    /// Log as much information as is available about the test run.
+    | Verbose
+
+/// Arguments controlling the test runner itself (not the tests therein).
+type Args =
+    {
+        /// The DLL containing the tests we'll reflectively discover and invoke.
+        Dll : FileInfo
+        /// If set, the output file into which we will write a TRX report. (We'll create parent directories as necessary.)
+        Trx : FileInfo option
+        /// Also contains the original string which specified the filter.
+        Filter : (string * Filter) option
+        /// How verbose to be with the test runner's own logging.
+        Logging : LogLevel
+        /// Maximum number of tests which can run concurrently. This setting overrides any LevelOfParallelism reflectively
+        /// extracted from the assembly under test.
+        LevelOfParallelism : int option
+        /// Abort if the test runner is running for longer than this timeout.
+        Timeout : TimeSpan option
+    }
+
+    /// Parse `argv` into a structured Args.
+    static member Parse (args : string list) : Args =
+        match args with
+        | [] -> failwith "The first arg must be a positional arg, the DLL to test."
+        | dll :: args ->
+
+        let rec go
+            (trx : FileInfo option)
+            (filter : (string * Filter) option)
+            (logging : LogLevel option)
+            (par : int option)
+            (timeout : TimeSpan option)
+            (args : string list)
+            =
+            match args with
+            | [] ->
+                {
+                    Dll = FileInfo dll
+                    Trx = trx
+                    Filter = filter
+                    Logging = logging |> Option.defaultValue LogLevel.Nothing
+                    LevelOfParallelism = par
+                    Timeout = timeout
+                }
+            | Key "--filter" filterStr :: rest
+            | "--filter" :: filterStr :: rest ->
+                match filter with
+                | Some _ -> failwith "Two conflicting filters; you can only specify --filter once"
+                | None -> go trx (Some (filterStr, Filter.parse filterStr)) logging par timeout rest
+            | Key "--trx" trxStr :: rest
+            | "--trx" :: trxStr :: rest ->
+                match trx with
+                | Some _ -> failwith "Two conflicting TRX outputs; you can only specify --trx once"
+                | None -> go (Some (FileInfo trxStr)) filter logging par timeout rest
+            | Key "--verbose" verboseStr :: rest
+            | "--verbose" :: verboseStr :: rest ->
+                match logging with
+                | Some _ -> failwith "Two conflicting --verbose outputs; you can only specify --verbose once"
+                | None ->
+                    let verbose =
+                        if Boolean.Parse verboseStr then
+                            LogLevel.Verbose
+                        else
+                            LogLevel.Nothing
+
+                    go trx filter (Some verbose) par timeout rest
+            | Key "--parallelism" parStr :: rest
+            | "--parallelism" :: parStr :: rest ->
+                match par with
+                | Some _ -> failwith "Two conflicting --parallelism outputs; you can only specify --parallelism once"
+                | None -> go trx filter logging (Some (Int32.Parse parStr)) timeout rest
+            | Key "--timeout-seconds" timeoutStr :: rest
+            | "--timeout-seconds" :: timeoutStr :: rest ->
+                match timeout with
+                | Some _ ->
+                    failwith "Two conflicting --timeout-seconds outputs; you can only specify --timeout-seconds once"
+                | None -> go trx filter logging par (Some (TimeSpan.FromSeconds (Int32.Parse timeoutStr |> float))) rest
+            | k :: _rest -> failwith $"Unrecognised arg %s{k}"
+
+        go None None None None None args

WoofWare.NUnitTestRunner.Lib/AssemblyLevelAttributes.fs

@@ -0,0 +1,58 @@
+namespace WoofWare.NUnitTestRunner
+
+open System.Reflection
+
+/// Attributes at the assembly level which control the behaviour of NUnit.
+type AssemblyLevelAttributes =
+    {
+        /// How many tests can be running at once, if anything's running in parallel.
+        Parallelism : int option
+        /// Whether the tests in this assembly can be parallelised at all.
+        Parallelizable : Parallelizable<AssemblyParallelScope> option
+    }
+
+[<RequireQualifiedAccess>]
+module AssemblyLevelAttributes =
+
+    /// Reflectively obtain the values of any relevant assembly attributes.
+    let get (assy : Assembly) : AssemblyLevelAttributes =
+        ((None, None), assy.CustomAttributes)
+        ||> Seq.fold (fun (levelPar, par) attr ->
+            match attr.AttributeType.FullName with
+            | "NUnit.Framework.LevelOfParallelismAttribute" ->
+                let arg = attr.ConstructorArguments |> Seq.exactlyOne |> _.Value |> unbox<int>
+
+                match levelPar with
+                | None -> (Some arg, par)
+                | Some existing ->
+                    failwith $"Assembly %s{assy.Location} declares parallelism %i{arg} and also %i{existing}"
+            | "NUnit.Framework.NonParallelizableAttribute" ->
+                match levelPar with
+                | None -> (Some 1, par)
+                | Some existing ->
+                    failwith
+                        $"Assembly %s{assy.Location} declares non-parallelizable and also parallelism %i{existing}"
+            | "NUnit.Framework.ParallelizableAttribute" ->
+                match par with
+                | Some _ -> failwith "Got multiple Parallelize attributes in assembly"
+                | None ->
+                    match attr.ConstructorArguments |> Seq.toList with
+                    | [] -> levelPar, Some (Parallelizable.Yes AssemblyParallelScope.Fixtures)
+                    | [ v ] ->
+                        match v.Value with
+                        | :? int as v ->
+                            match v with
+                            | 512 -> levelPar, Some (Parallelizable.Yes AssemblyParallelScope.Fixtures)
+                            | 256 -> levelPar, Some (Parallelizable.Yes AssemblyParallelScope.Children)
+                            | 257 -> failwith "ParallelScope.All is invalid on assemblies; only Fixtures or Children"
+                            | 1 -> failwith "ParallelScope.Self is invalid on assemblies; only Fixtures or Children"
+                            | v -> failwith $"Could not recognise value %i{v} of parallel scope on assembly"
+                        | v -> failwith $"Unexpectedly non-int value %O{v} of parallel scope on assembly"
+                    | _ -> failwith "unexpectedly got multiple args to Parallelizable on assembly"
+            | _ -> levelPar, par
+        )
+        |> fun (par, canPar) ->
+            {
+                Parallelizable = canPar
+                Parallelism = par
+            }

WoofWare.NUnitTestRunner.Lib/Context.fs

@@ -0,0 +1,180 @@
+namespace WoofWare.NUnitTestRunner
+
+open System
+open System.Collections.Generic
+open System.IO
+open System.Reflection
+open System.Runtime.Loader
+open System.Text
+open System.Threading
+
+type internal OutputStreamId = | OutputStreamId of Guid
+
+type private ThreadAwareWriter
+    (
+        local : AsyncLocal<OutputStreamId>,
+        underlying : Dictionary<OutputStreamId, TextWriter>,
+        mem : Dictionary<OutputStreamId, MemoryStream>
+    )
+    =
+    inherit TextWriter ()
+    override _.get_Encoding () = Encoding.Default
+
+    override this.Write (v : char) : unit =
+        use prev = ExecutionContext.Capture ()
+
+        (fun _ ->
+            (fun () ->
+                match underlying.TryGetValue local.Value with
+                | true, output -> output.Write v
+                | false, _ ->
+                    let wanted =
+                        underlying |> Seq.map (fun (KeyValue (a, b)) -> $"%O{a}") |> String.concat "\n"
+
+                    failwith $"no such context: %O{local.Value}\nwanted:\n"
+            )
+            |> lock underlying
+        )
+        |> fun action -> ExecutionContext.Run (prev, action, ())
+
+    override this.WriteLine (v : string) : unit =
+        use prev = ExecutionContext.Capture ()
+
+        (fun _ ->
+            (fun () ->
+                match underlying.TryGetValue local.Value with
+                | true, output -> output.WriteLine v
+                | false, _ ->
+                    let wanted =
+                        underlying |> Seq.map (fun (KeyValue (a, b)) -> $"%O{a}") |> String.concat "\n"
+
+                    failwith $"no such context: %O{local.Value}\nwanted:\n"
+            )
+            |> lock underlying
+        )
+        |> fun action -> ExecutionContext.Run (prev, action, ())
+
+/// Wraps up the necessary context to intercept global state.
+type TestContexts =
+    private
+        {
+            /// Accesses to this must be locked on StdOutWriters.
+            StdOuts : Dictionary<OutputStreamId, MemoryStream>
+            /// Accesses to this must be locked on StdErrWriters.
+            StdErrs : Dictionary<OutputStreamId, MemoryStream>
+            StdOutWriters : Dictionary<OutputStreamId, TextWriter>
+            StdErrWriters : Dictionary<OutputStreamId, TextWriter>
+            StdOutWriter : TextWriter
+            StdErrWriter : TextWriter
+            AsyncLocal : AsyncLocal<OutputStreamId>
+        }
+
+    /// Call this exactly once.
+    static member Empty () =
+        let stdouts = Dictionary ()
+        let stderrs = Dictionary ()
+        let stdoutWriters = Dictionary ()
+        let stderrWriters = Dictionary ()
+        let local = AsyncLocal ()
+        let stdoutWriter = new ThreadAwareWriter (local, stdoutWriters, stdouts)
+        let stderrWriter = new ThreadAwareWriter (local, stderrWriters, stderrs)
+
+        {
+            StdOuts = stdouts
+            StdErrs = stderrs
+            StdOutWriter = stdoutWriter
+            StdErrWriter = stderrWriter
+            StdOutWriters = stdoutWriters
+            StdErrWriters = stderrWriters
+            AsyncLocal = local
+        }
+
+    member internal this.Stdout : TextWriter = this.StdOutWriter
+    member internal this.Stderr : TextWriter = this.StdErrWriter
+
+    member internal this.DumpStdout (id : OutputStreamId) : string =
+        lock
+            this.StdOutWriters
+            (fun () ->
+                this.StdOutWriters.[id].Flush ()
+                this.StdOuts.[id].ToArray ()
+            )
+        |> Encoding.Default.GetString
+
+    member internal this.DumpStderr (id : OutputStreamId) : string =
+        lock
+            this.StdErrWriters
+            (fun () ->
+                this.StdErrWriters.[id].Flush ()
+                this.StdErrs.[id].ToArray ()
+            )
+        |> Encoding.Default.GetString
+
+    member internal this.NewOutputs () =
+        let id = Guid.NewGuid () |> OutputStreamId
+        let msOut = new MemoryStream ()
+        let wrOut = new StreamWriter (msOut)
+        let msErr = new MemoryStream ()
+        let wrErr = new StreamWriter (msErr)
+
+        lock
+            this.StdOutWriters
+            (fun () ->
+                this.StdOutWriters.Add (id, wrOut)
+                this.StdOuts.Add (id, msOut)
+            )
+
+        lock
+            this.StdErrWriters
+            (fun () ->
+                this.StdErrWriters.Add (id, wrErr)
+                this.StdErrs.Add (id, msErr)
+            )
+
+        id
+
+    interface IDisposable with
+        member this.Dispose () =
+            // TODO: dispose the streams
+            ()
+
+/// A separate AssemblyLoadContext within which you can run the tests in the given DLL.
+/// Supply places to find the .NET runtimes.
+type LoadContext (dll : FileInfo, runtimes : DirectoryInfo list, contexts : TestContexts) =
+    inherit AssemblyLoadContext ()
+
+    /// Load the assembly with the given name into this assembly context.
+    /// This additionally monkey-patches System.Console: it performs SetOut and SetError on them
+    /// so that they redirect their outputs into the given `TestContexts`.
+    override this.Load (target : AssemblyName) : Assembly =
+        let path = Path.Combine (dll.Directory.FullName, $"%s{target.Name}.dll")
+
+        let assy =
+            if File.Exists path then
+                this.LoadFromAssemblyPath path
+            else
+
+            runtimes
+            |> List.tryPick (fun di ->
+                let path = Path.Combine (di.FullName, $"%s{target.Name}.dll")
+
+                if File.Exists path then
+                    this.LoadFromAssemblyPath path |> Some
+                else
+                    None
+            )
+            |> Option.defaultValue null
+
+        if target.Name = "System.Console" then
+            if isNull assy then
+                failwith "could not monkey-patch System.Console"
+            else
+                let consoleType = assy.GetType "System.Console"
+                let setOut = consoleType.GetMethod "SetOut"
+                setOut.Invoke ((null : obj), [| contexts.Stdout |]) |> unbox<unit>
+                let setErr = consoleType.GetMethod "SetError"
+                setErr.Invoke ((null : obj), [| contexts.Stderr |]) |> unbox<unit>
+
+            assy
+        else
+            assy

WoofWare.NUnitTestRunner.Lib/CreateTrxReport.fs

@@ -0,0 +1,243 @@
+namespace WoofWare.NUnitTestRunner
+
+open System
+open System.Reflection
+
+/// Methods for constructing TRX reports.
+[<RequireQualifiedAccess>]
+module BuildTrxReport =
+
+    /// Build a TRX report from the given results.
+    let build
+        (assy : Assembly)
+        (creationTime : DateTimeOffset)
+        (startTime : DateTimeOffset)
+        (results : FixtureRunResults list)
+        : TrxReport
+        =
+        let finishTime = DateTimeOffset.Now
+        let finishTimeHumanReadable = finishTime.ToString @"yyyy-MM-dd HH:mm:ss"
+        let nowMachine = finishTime.ToString @"yyyy-MM-dd_HH_mm_ss"
+
+        let testListId = Guid.NewGuid ()
+
+        let testDefinitions, testEntries =
+            results
+            |> List.collect (fun results -> results.IndividualTestRunMetadata)
+            |> List.map (fun (data, _) ->
+                let defn =
+                    {
+                        Name = data.TestName
+                        Storage = assy.Location.ToLowerInvariant ()
+                        Id = data.TestId
+                        Execution =
+                            {
+                                Id = data.ExecutionId
+                            }
+                        TestMethod =
+                            {
+                                CodeBase = assy.Location
+                                AdapterTypeName = Uri "executor://woofware/"
+                                ClassName = data.ClassName
+                                Name = data.TestName
+                            }
+                    }
+
+                let entry : TrxTestEntry =
+                    {
+                        TestListId = testListId
+                        ExecutionId = data.ExecutionId
+                        TestId = data.TestId
+
+                    }
+
+                defn, entry
+            )
+            |> List.unzip
+
+        let hostname = Environment.MachineName
+
+        let settings =
+            {
+                Name = "default"
+                Id = Guid.NewGuid ()
+                Deployment =
+                    {
+                        RunDeploymentRoot = $"_%s{hostname}_%s{nowMachine}"
+                    }
+            }
+
+        let testList : TrxTestListEntry =
+            {
+                Id = testListId
+                Name = "All"
+            }
+
+        let counters =
+            (TrxCounters.Zero, results)
+            // TODO: this is woefully inefficient
+            ||> List.fold (fun counters results ->
+                let counters =
+                    (counters, results.Failed)
+                    ||> List.fold (fun counters (_, _) ->
+                        // TODO: the counters can be more specific about the failure mode
+                        counters.AddFailed ()
+                    )
+
+                let counters =
+                    (counters, results.OtherFailures)
+                    ||> List.fold (fun counters _ ->
+                        // TODO: the counters can be more specific about the failure mode
+                        counters.AddFailed ()
+                    )
+
+                (counters, results.Success)
+                ||> List.fold (fun counters (_, success, _) ->
+                    match success with
+                    | TestMemberSuccess.Ok -> counters.AddPassed ()
+                    | TestMemberSuccess.Ignored _
+                    | TestMemberSuccess.Explicit _ -> counters.AddNotExecuted ()
+                    | TestMemberSuccess.Inconclusive _ -> counters.AddInconclusive ()
+                )
+            )
+
+        // TODO: I'm sure we can do better than this; there's a whole range of possible
+        // states!
+        let outcome =
+            if counters.Failed > 0u then
+                TrxOutcome.Failed
+            else
+                TrxOutcome.Completed
+
+        let resultSummary : TrxResultsSummary =
+            {
+                Outcome = outcome
+                Counters = counters
+                Output =
+                    {
+                        StdOut = None
+                        StdErr = None
+                        ErrorInfo = None
+                    }
+                RunInfos =
+                    [
+                    // TODO: capture stdout
+                    ]
+            }
+
+        let times : TrxReportTimes =
+            {
+                Creation = creationTime
+                Queuing = startTime
+                Start = startTime
+                Finish = finishTime
+
+            }
+
+        let magicGuid = Guid.Parse "13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b"
+
+        let results =
+            results
+            |> List.collect (fun results -> results.IndividualTestRunMetadata)
+            |> List.map (fun (i, cause) ->
+                let exc =
+                    match cause with
+                    | Choice2Of3 _ -> None
+                    | Choice1Of3 (TestMemberFailure.Malformed reasons) ->
+                        {
+                            StackTrace = None
+                            Message = reasons |> String.concat "\n" |> Some
+                        }
+                        |> Some
+                    | Choice1Of3 (TestMemberFailure.Failed fail)
+                    | Choice1Of3 (TestMemberFailure.Failed fail)
+                    | Choice1Of3 (TestMemberFailure.Failed fail) ->
+                        ((None, None), fail)
+                        ||> List.fold (fun (stackTrace, message) tf ->
+                            match tf with
+                            | TestFailure.TestFailed (UserMethodFailure.Threw (_, exc))
+                            | TestFailure.SetUpFailed (UserMethodFailure.Threw (_, exc))
+                            | TestFailure.TearDownFailed (UserMethodFailure.Threw (_, exc)) ->
+                                let stackTrace =
+                                    match stackTrace with
+                                    | None -> (exc : Exception).ToString ()
+                                    | Some s -> s
+
+                                (Some stackTrace, message)
+                            | TestFailure.TestFailed (UserMethodFailure.ReturnedNonUnit (_, ret))
+                            | TestFailure.SetUpFailed (UserMethodFailure.ReturnedNonUnit (_, ret))
+                            | TestFailure.TearDownFailed (UserMethodFailure.ReturnedNonUnit (_, ret)) ->
+                                let newMessage = $"returned non-unit value %O{ret}"
+
+                                let message =
+                                    match message with
+                                    | None -> newMessage
+                                    | Some message -> $"%s{message}\n%s{newMessage}"
+
+                                (stackTrace, Some message)
+                        )
+                        |> fun (stackTrace, message) ->
+                            {
+                                StackTrace = stackTrace
+                                Message = message
+                            }
+                            |> Some
+                    | Choice3Of3 (UserMethodFailure.Threw (_, exc)) ->
+                        {
+                            StackTrace = (exc : Exception).ToString () |> Some
+                            Message = None
+                        }
+                        |> Some
+                    | Choice3Of3 (UserMethodFailure.ReturnedNonUnit (_, ret)) ->
+                        {
+                            Message = $"returned non-unit value %O{ret}" |> Some
+                            StackTrace = None
+                        }
+                        |> Some
+
+                let outcome =
+                    match cause with
+                    | Choice1Of3 _ -> TrxTestOutcome.Failed
+                    | Choice2Of3 TestMemberSuccess.Ok -> TrxTestOutcome.Passed
+                    | Choice2Of3 (TestMemberSuccess.Inconclusive _) -> TrxTestOutcome.Inconclusive
+                    | Choice2Of3 (TestMemberSuccess.Ignored _)
+                    | Choice2Of3 (TestMemberSuccess.Explicit _) -> TrxTestOutcome.NotExecuted
+                    // TODO: we can totally do better here, more fine-grained classification
+                    | Choice3Of3 _ -> TrxTestOutcome.Failed
+
+                {
+                    ExecutionId = i.ExecutionId
+                    TestId = i.TestId
+                    TestName = i.TestName
+                    ComputerName = i.ComputerName
+                    Duration = i.End - i.Start
+                    StartTime = i.Start
+                    EndTime = i.End
+                    TestType = magicGuid
+                    Outcome = outcome
+                    TestListId = testListId
+                    RelativeResultsDirectory = i.ExecutionId.ToString () // for some reason
+                    Output =
+                        match i.StdOut, i.StdErr, exc with
+                        | None, None, None -> None
+                        | stdout, stderr, exc ->
+                            Some
+                                {
+                                    TrxOutput.StdOut = stdout
+                                    StdErr = stderr
+                                    ErrorInfo = exc
+                                }
+                }
+            )
+
+        {
+            Id = Guid.NewGuid ()
+            Name = $"@%s{hostname} %s{finishTimeHumanReadable}"
+            Times = times
+            Settings = settings
+            Results = results
+            TestDefinitions = testDefinitions
+            TestEntries = testEntries
+            TestLists = [ testList ]
+            ResultsSummary = resultSummary
+        }

WoofWare.NUnitTestRunner.Lib/Domain.fs

@@ -88,6 +88,7 @@ type SingleTestMethod =
 
 /// A test fixture (usually represented by the [<TestFixture>]` attribute), which may contain many tests,
 /// each of which may run many times.
+[<NoComparison>]
 type TestFixture =
     {
         /// The assembly which contains this TestFixture, loaded into a separate context.

WoofWare.NUnitTestRunner.Lib/DotnetRuntime.fs

@@ -7,11 +7,7 @@ open WoofWare.DotnetRuntimeLocator
 /// Functions for locating .NET runtimes.
 [<RequireQualifiedAccess>]
 module DotnetRuntime =
-    let private selectRuntime
-        (config : RuntimeOptions)
-        (f : DotnetEnvironmentInfo)
-        : Choice<DotnetEnvironmentFrameworkInfo, DotnetEnvironmentSdkInfo> option
-        =
+    let private selectRuntime (config : RuntimeOptions) (f : DotnetEnvironmentInfo) : DirectoryInfo list =
         let rollForward =
             match Environment.GetEnvironmentVariable "DOTNET_ROLL_FORWARD" with
             | null ->
@@ -20,6 +16,14 @@ module DotnetRuntime =
                 |> Option.defaultValue RollForward.Minor
             | s -> RollForward.Parse s
 
+        if
+            Option.isSome config.IncludedFramework
+            || Option.isSome config.IncludedFrameworks
+        then
+            // No need for a framework that's anywhere other than the given DLL.
+            []
+        else
+
         let desiredVersions =
             match config.Framework with
             | Some f -> [ Version f.Version, f.Name ]
@@ -66,15 +70,13 @@ module DotnetRuntime =
 
                     name, data.Installed
                 )
-                // TODO: how do we select between many available frameworks?
-                |> Seq.tryHead
+                |> Seq.toList
 
-            match available with
-            | Some (_, f) -> Some (Choice1Of2 f)
-            | None ->
-                // TODO: maybe we can ask the SDK. But we keep on trucking: maybe we're self-contained,
-                // and we'll actually find all the runtime next to the DLL.
-                None
+            // TODO: maybe we can ask the SDK if we don't have any runtimes.
+            // But we keep on trucking: maybe we're self-contained, and we'll actually find all the runtime next to the
+            // DLL.
+            available
+            |> List.map (fun (_name, runtime) -> DirectoryInfo $"%s{runtime.Path}/%s{runtime.Version}")
         | _ -> failwith "non-minor RollForward not supported yet; please shout if you want it"
 
     /// Given an executable DLL, locate the .NET runtime that can best run it.
@@ -96,9 +98,4 @@ module DotnetRuntime =
 
         let runtime = selectRuntime runtimeConfig availableRuntimes
 
-        match runtime with
-        | None ->
-            // Keep on trucking: let's be optimistic and hope that we're self-contained.
-            [ dll.Directory ]
-        | Some (Choice1Of2 runtime) -> [ dll.Directory ; DirectoryInfo $"%s{runtime.Path}/%s{runtime.Version}" ]
-        | Some (Choice2Of2 sdk) -> [ dll.Directory ; DirectoryInfo sdk.Path ]
+        dll.Directory :: runtime

WoofWare.NUnitTestRunner.Lib/Filter.fs

@@ -2,7 +2,7 @@ namespace WoofWare.NUnitTestRunner
 
 open System
 open System.IO
-open PrattParser
+open WoofWare.PrattParser
 
 // Documentation:
 // https://learn.microsoft.com/en-us/dotnet/core/testing/selective-unit-tests?pivots=mstest

WoofWare.NUnitTestRunner.Lib/ParallelQueue.fs

@@ -0,0 +1,422 @@
+namespace WoofWare.NUnitTestRunner
+
+open System
+open System.Threading
+open System.Threading.Tasks
+
+type private ThunkEvaluator<'ret> =
+    abstract Eval<'a> : (unit -> 'a) -> AsyncReplyChannel<'a> -> 'ret
+
+type private ThunkCrate =
+    abstract Apply<'ret> : ThunkEvaluator<'ret> -> 'ret
+
+[<RequireQualifiedAccess>]
+module private ThunkCrate =
+    let make<'a> (t : unit -> 'a) (rc : AsyncReplyChannel<'a>) : ThunkCrate =
+        { new ThunkCrate with
+            member _.Apply e = e.Eval t rc
+        }
+
+type private FakeUnit = FakeUnit
+
+/// A handle to a running test fixture.
+type TestFixtureRunningToken = private | TestFixtureRunningToken of TestFixture
+
+/// A handle to a test fixture whose setup method has been called.
+type TestFixtureSetupToken = private | TestFixtureSetupToken of TestFixture
+
+[<RequireQualifiedAccess>]
+module private TestFixtureSetupToken =
+    let vouchNoSetupRequired (TestFixtureRunningToken tf) = TestFixtureSetupToken tf
+
+/// A handle to a test fixture whose setup method has been called.
+type TestFixtureTearDownToken = private | TestFixtureTearDownToken of TestFixture
+
+[<RequireQualifiedAccess>]
+module private TestFixtureTearDownToken =
+    let vouchNoTearDownRequired (TestFixtureSetupToken tf) = TestFixtureTearDownToken tf
+
+[<RequireQualifiedAccess>]
+type private MailboxMessage =
+    | Quit of AsyncReplyChannel<unit>
+    /// Check current state, see if we need to start more tests, etc.
+    | Reconcile
+    | RunTest of within : TestFixture * Parallelizable<unit> option * test : ThunkCrate
+    | BeginTestFixture of TestFixture * AsyncReplyChannel<TestFixtureRunningToken>
+    | EndTestFixture of TestFixtureTearDownToken * AsyncReplyChannel<unit>
+
+type private RunningFixture =
+    {
+        Fixture : TestFixture
+        RunningCanParallelize : bool
+        Running : Task list
+        Waiting : ((unit -> Task) * Parallelizable<unit> option) list
+    }
+
+    static member Make (f : TestFixture) =
+        {
+            Fixture = f
+            Running = []
+            RunningCanParallelize = true
+            Waiting = []
+        }
+
+type private RunningState =
+    {
+        MaxParallelism : int
+        // TODO: make these efficiently look-up-able
+        CurrentlyRunning : RunningFixture list
+        Waiting : (TestFixture * AsyncReplyChannel<TestFixtureRunningToken>) list
+    }
+
+    member this.NewTest (tf : TestFixture) (par : Parallelizable<unit> option) (test : unit -> Task) =
+        {
+            MaxParallelism = this.MaxParallelism
+            Waiting = this.Waiting
+            CurrentlyRunning =
+                let found = ref 0
+
+                this.CurrentlyRunning
+                |> List.map (fun f ->
+                    if Object.ReferenceEquals (f.Fixture, tf) then
+                        Interlocked.Increment found |> ignore<int>
+
+                        { f with
+                            Waiting = (test, par) :: f.Waiting
+                        }
+                    else
+                        f
+                )
+                |> fun l ->
+                    match found.Value with
+                    | 1 -> l
+                    | 0 -> failwith $"Unexpectedly did not find the running test fixture '%s{tf.Name}' to add a test to"
+                    | _ -> failwith $"Unexpectedly found the running test fixture '%s{tf.Name}' multiple times in list"
+        }
+
+    member this.CompleteFixture (tf : TestFixture) : RunningState =
+        let rec go (acc : RunningFixture list) (running : RunningFixture list) =
+            match running with
+            | [] -> failwith "Caller has somehow called EndTestFixture while we're not running that test fixture"
+            | runningFixture :: tail ->
+                if Object.ReferenceEquals (runningFixture.Fixture, tf) then
+                    match runningFixture.Running, runningFixture.Waiting with
+                    | [], [] -> acc @ tail
+                    | r, [] ->
+                        failwith $"Caller has called EndTestFixture while its tests are still running (%i{r.Length})"
+                    | [], r ->
+                        failwith $"Caller has called EndTestFixture while it has tests waiting to run (%i{r.Length})"
+                    | r, s ->
+                        failwith
+                            $"Caller has called EndTestFixture while it has tests waiting to run (%i{s.Length}) and test running (%i{r.Length})"
+                else
+                    go (runningFixture :: acc) tail
+
+        let currentlyRunning = go [] this.CurrentlyRunning
+
+        {
+            CurrentlyRunning = currentlyRunning
+            Waiting = this.Waiting
+            MaxParallelism = this.MaxParallelism
+        }
+
+type private MailboxState =
+    | Idle
+    | Running of RunningState
+
+/// Run some things in parallel.
+/// TODO: actually implement the parallelism! Right now this just runs everything serially.
+/// TODO: consume the cancellation token
+type ParallelQueue
+    (parallelism : int option, _scope : Parallelizable<AssemblyParallelScope> option, ?ct : CancellationToken)
+    =
+    let parallelism =
+        match parallelism with
+        | None -> max (Environment.ProcessorCount / 2) 2
+        | Some p -> p
+
+    let rec processTask (state : MailboxState) (m : MailboxProcessor<MailboxMessage>) =
+        async {
+            let! message = m.Receive ()
+
+            match message with
+            | MailboxMessage.Quit rc -> rc.Reply ()
+            | MailboxMessage.Reconcile ->
+                match state with
+                | Idle -> return! processTask state m
+                | Running r ->
+
+                match r.CurrentlyRunning with
+                | [] ->
+                    match r.Waiting with
+                    | [] -> return! processTask Idle m
+                    | (head, rc) :: tail ->
+                        rc.Reply (TestFixtureRunningToken head)
+
+                        let newRunning =
+                            {
+                                Fixture = head
+                                Running = []
+                                RunningCanParallelize = true
+                                Waiting = []
+                            }
+
+                        let state =
+                            {
+                                MaxParallelism = r.MaxParallelism
+                                CurrentlyRunning = [ newRunning ]
+                                Waiting = tail
+                            }
+                        // For now, we'll just run one fixture at a time. When we run multiple fixtures in parallel,
+                        // we probably want to call Reconcile here again.
+                        return! processTask (Running state) m
+                | [ currentlyRunning ] ->
+                    let currentlyRunningTasks =
+                        currentlyRunning.Running |> List.filter (fun t -> not t.IsCompleted)
+
+                    let r =
+                        { r with
+                            CurrentlyRunning =
+                                [
+                                    { currentlyRunning with
+                                        Running = currentlyRunningTasks
+                                    }
+                                ]
+                        }
+
+                    match currentlyRunningTasks with
+                    | [] ->
+                        match currentlyRunning.Waiting with
+                        | [] ->
+                            // Nothing to run yet
+                            return! processTask (Running r) m
+                        | (head, par) :: tail ->
+                            let par =
+                                match par with
+                                | None -> true
+                                | Some Parallelizable.No -> false
+                                | Some (Parallelizable.Yes ()) -> true
+
+                            let state =
+                                {
+                                    Fixture = currentlyRunning.Fixture
+                                    RunningCanParallelize = par
+                                    Waiting = tail
+                                    Running = [ head () ]
+                                }
+
+                            m.Post MailboxMessage.Reconcile
+
+                            return!
+                                processTask
+                                    (Running
+                                        { r with
+                                            CurrentlyRunning = [ state ]
+                                        })
+                                    m
+
+                    | currentlyRunningTasks ->
+
+                    if currentlyRunningTasks.Length >= parallelism then
+                        return! processTask (Running r) m
+                    else
+
+                    match currentlyRunning.Waiting, currentlyRunning.RunningCanParallelize with
+                    | [], _ ->
+                        // No new candidates.
+                        return! processTask (Running r) m
+                    | _, false ->
+                        // The running test(s) can't have others added.
+                        return! processTask (Running r) m
+                    | (head, par) :: tail, true ->
+                        match par with
+                        | Some Parallelizable.No -> return! processTask (Running r) m
+                        | Some (Parallelizable.Yes ()) ->
+                            let state =
+                                {
+                                    RunningState.MaxParallelism = r.MaxParallelism
+                                    Waiting = r.Waiting
+                                    CurrentlyRunning =
+                                        [
+                                            {
+                                                Fixture = currentlyRunning.Fixture
+                                                RunningCanParallelize = true
+                                                Running = head () :: currentlyRunning.Running
+                                                Waiting = tail
+                                            }
+                                        ]
+                                }
+
+                            m.Post MailboxMessage.Reconcile
+                            return! processTask (Running state) m
+                        | None ->
+                            match currentlyRunning.Fixture.Parallelize with
+                            | Some Parallelizable.No
+                            | Some (Parallelizable.Yes ClassParallelScope.Self)
+                            | Some (Parallelizable.Yes ClassParallelScope.Fixtures) ->
+                                // Can't add this test to the parallel queue right now
+                                return! processTask (Running r) m
+                            | None
+                            | Some (Parallelizable.Yes ClassParallelScope.All)
+                            | Some (Parallelizable.Yes ClassParallelScope.Children) ->
+                                let state =
+                                    {
+                                        Fixture = currentlyRunning.Fixture
+                                        RunningCanParallelize = true
+                                        Waiting = tail
+                                        Running = (head ()) :: currentlyRunningTasks
+                                    }
+
+                                m.Post MailboxMessage.Reconcile
+
+                                return!
+                                    processTask
+                                        (Running
+                                            { r with
+                                                CurrentlyRunning = [ state ]
+                                            })
+                                        m
+                | _ -> failwith "Logic error: we currently only run one fixture at a time"
+            | MailboxMessage.BeginTestFixture (tf, rc) ->
+                match state with
+                | Running state ->
+                    let state =
+                        {
+                            MaxParallelism = state.MaxParallelism
+                            CurrentlyRunning = state.CurrentlyRunning
+                            Waiting = (tf, rc) :: state.Waiting
+                        }
+                        |> Running
+
+                    m.Post MailboxMessage.Reconcile
+                    return! processTask state m
+                | Idle ->
+                    let state =
+                        {
+                            MaxParallelism = parallelism
+                            CurrentlyRunning = []
+                            Waiting = [ (tf, rc) ]
+                        }
+                        |> Running
+
+                    m.Post MailboxMessage.Reconcile
+                    return! processTask state m
+            | MailboxMessage.EndTestFixture (TestFixtureTearDownToken tf, rc) ->
+                match state with
+                | Idle ->
+                    return failwith "Caller has somehow called EndTestFixture while we're not running a test fixture"
+                | Running state ->
+                    let state = state.CompleteFixture tf
+                    rc.Reply ()
+                    m.Post MailboxMessage.Reconcile
+                    return! processTask (Running state) m
+            | MailboxMessage.RunTest (withinFixture, par, message) ->
+                let t () =
+                    { new ThunkEvaluator<_> with
+                        member _.Eval<'b> (t : unit -> 'b) rc =
+                            let tcs = TaskCompletionSource ()
+                            use ec = ExecutionContext.Capture ()
+
+                            fun () ->
+                                ExecutionContext.Run (
+                                    ec,
+                                    (fun _ ->
+                                        let result = t ()
+                                        tcs.SetResult ()
+                                        m.Post MailboxMessage.Reconcile
+                                        rc.Reply result
+                                    ),
+                                    ()
+                                )
+                            |> Task.Factory.StartNew
+                            |> ignore<Task>
+
+                            tcs.Task
+                    }
+                    |> message.Apply
+
+                let state =
+                    match state with
+                    | Idle -> failwith "somehow asked the queue to run tests when there is no active fixture"
+                    | Running state -> state.NewTest withinFixture par t |> Running
+
+                m.Post MailboxMessage.Reconcile
+
+                return! processTask state m
+        }
+
+    let mb = new MailboxProcessor<_> (processTask MailboxState.Idle)
+    do mb.Start ()
+
+    /// Request to run the given action, freely in parallel with other running tests.
+    /// The resulting Task will return when the action has completed.
+    member _.Run<'a>
+        (TestFixtureSetupToken parent)
+        (scope : Parallelizable<unit> option)
+        (action : unit -> 'a)
+        : 'a Task
+        =
+        (fun rc -> MailboxMessage.RunTest (parent, scope, ThunkCrate.make action rc))
+        |> mb.PostAndAsyncReply
+        |> Async.StartAsTask
+
+    /// Declare that we wish to start the given test fixture. The resulting Task will return
+    /// when you are allowed to start running tests from that fixture.
+    /// Once you've finished running tests from that fixture, call EndTestFixture.
+    member _.StartTestFixture (tf : TestFixture) : Task<TestFixtureRunningToken> =
+        fun rc -> MailboxMessage.BeginTestFixture (tf, rc)
+        |> mb.PostAndAsyncReply
+        |> Async.StartAsTask
+
+    /// Run the given one-time setup for the test fixture.
+    member _.RunTestSetup (TestFixtureRunningToken parent) (action : unit -> 'a) : ('a * TestFixtureSetupToken) Task =
+        task {
+            let par =
+                parent.Parallelize
+                |> Option.map (fun p ->
+                    match p with
+                    | Parallelizable.No -> Parallelizable.No
+                    | Parallelizable.Yes _ -> Parallelizable.Yes ()
+                )
+
+            let! response =
+                (fun rc -> MailboxMessage.RunTest (parent, par, ThunkCrate.make action rc))
+                |> mb.PostAndAsyncReply
+
+            return response, TestFixtureSetupToken parent
+        }
+
+    /// Run the given one-time tear-down for the test fixture.
+    member _.RunTestTearDown
+        (TestFixtureSetupToken parent)
+        (action : unit -> 'a)
+        : ('a * TestFixtureTearDownToken) Task
+        =
+        task {
+            let par =
+                parent.Parallelize
+                |> Option.map (fun p ->
+                    match p with
+                    | Parallelizable.No -> Parallelizable.No
+                    | Parallelizable.Yes _ -> Parallelizable.Yes ()
+                )
+
+            let! response =
+                (fun rc -> MailboxMessage.RunTest (parent, par, ThunkCrate.make action rc))
+                |> mb.PostAndAsyncReply
+
+            return response, TestFixtureTearDownToken parent
+        }
+
+    /// Declare that we have finished submitting requests to run in the given test fixture.
+    /// You don't need to worry about when the resulting Task returns, but we provide it just in case.
+    member _.EndTestFixture (tf : TestFixtureTearDownToken) : Task<unit> =
+        (fun rc -> MailboxMessage.EndTestFixture (tf, rc))
+        |> mb.PostAndAsyncReply
+        |> Async.StartAsTask
+
+    interface IDisposable with
+        member _.Dispose () =
+            // Still race conditions, of course: people could still be submitting after we finish the sync.
+            mb.PostAndReply MailboxMessage.Quit
+            (mb :> IDisposable).Dispose ()

WoofWare.NUnitTestRunner.Lib/RuntimeConfig.fs

@@ -16,6 +16,8 @@ type internal RuntimeOptions =
         Tfm : string
         Framework : FrameworkDescription option
         Frameworks : FrameworkDescription list option
+        IncludedFramework : FrameworkDescription option
+        IncludedFrameworks : FrameworkDescription list option
         RollForward : string option
     }
 

WoofWare.NUnitTestRunner.Lib/SurfaceBaseline.txt

@@ -1,3 +1,26 @@
+WoofWare.NUnitTestRunner.Args inherit obj, implements WoofWare.NUnitTestRunner.Args System.IEquatable, System.Collections.IStructuralEquatable
+WoofWare.NUnitTestRunner.Args..ctor [constructor]: (System.IO.FileInfo, System.IO.FileInfo option, (string * WoofWare.NUnitTestRunner.Filter) option, WoofWare.NUnitTestRunner.LogLevel, int option, System.TimeSpan option)
+WoofWare.NUnitTestRunner.Args.Dll [property]: [read-only] System.IO.FileInfo
+WoofWare.NUnitTestRunner.Args.Filter [property]: [read-only] (string * WoofWare.NUnitTestRunner.Filter) option
+WoofWare.NUnitTestRunner.Args.get_Dll [method]: unit -> System.IO.FileInfo
+WoofWare.NUnitTestRunner.Args.get_Filter [method]: unit -> (string * WoofWare.NUnitTestRunner.Filter) option
+WoofWare.NUnitTestRunner.Args.get_LevelOfParallelism [method]: unit -> int option
+WoofWare.NUnitTestRunner.Args.get_Logging [method]: unit -> WoofWare.NUnitTestRunner.LogLevel
+WoofWare.NUnitTestRunner.Args.get_Timeout [method]: unit -> System.TimeSpan option
+WoofWare.NUnitTestRunner.Args.get_Trx [method]: unit -> System.IO.FileInfo option
+WoofWare.NUnitTestRunner.Args.LevelOfParallelism [property]: [read-only] int option
+WoofWare.NUnitTestRunner.Args.Logging [property]: [read-only] WoofWare.NUnitTestRunner.LogLevel
+WoofWare.NUnitTestRunner.Args.Parse [static method]: string list -> WoofWare.NUnitTestRunner.Args
+WoofWare.NUnitTestRunner.Args.Timeout [property]: [read-only] System.TimeSpan option
+WoofWare.NUnitTestRunner.Args.Trx [property]: [read-only] System.IO.FileInfo option
+WoofWare.NUnitTestRunner.AssemblyLevelAttributes inherit obj, implements WoofWare.NUnitTestRunner.AssemblyLevelAttributes System.IEquatable, System.Collections.IStructuralEquatable, WoofWare.NUnitTestRunner.AssemblyLevelAttributes System.IComparable, System.IComparable, System.Collections.IStructuralComparable
+WoofWare.NUnitTestRunner.AssemblyLevelAttributes..ctor [constructor]: (int option, WoofWare.NUnitTestRunner.AssemblyParallelScope WoofWare.NUnitTestRunner.Parallelizable option)
+WoofWare.NUnitTestRunner.AssemblyLevelAttributes.get_Parallelism [method]: unit -> int option
+WoofWare.NUnitTestRunner.AssemblyLevelAttributes.get_Parallelizable [method]: unit -> WoofWare.NUnitTestRunner.AssemblyParallelScope WoofWare.NUnitTestRunner.Parallelizable option
+WoofWare.NUnitTestRunner.AssemblyLevelAttributes.Parallelism [property]: [read-only] int option
+WoofWare.NUnitTestRunner.AssemblyLevelAttributes.Parallelizable [property]: [read-only] WoofWare.NUnitTestRunner.AssemblyParallelScope WoofWare.NUnitTestRunner.Parallelizable option
+WoofWare.NUnitTestRunner.AssemblyLevelAttributesModule inherit obj
+WoofWare.NUnitTestRunner.AssemblyLevelAttributesModule.get [static method]: System.Reflection.Assembly -> WoofWare.NUnitTestRunner.AssemblyLevelAttributes
 WoofWare.NUnitTestRunner.AssemblyParallelScope inherit obj, implements WoofWare.NUnitTestRunner.AssemblyParallelScope System.IEquatable, System.Collections.IStructuralEquatable, WoofWare.NUnitTestRunner.AssemblyParallelScope System.IComparable, System.IComparable, System.Collections.IStructuralComparable - union type with 2 cases
 WoofWare.NUnitTestRunner.AssemblyParallelScope+Tags inherit obj
 WoofWare.NUnitTestRunner.AssemblyParallelScope+Tags.Children [static field]: int = 0
@@ -12,6 +35,8 @@ WoofWare.NUnitTestRunner.AssemblyParallelScope.get_Tag [method]: unit -> int
 WoofWare.NUnitTestRunner.AssemblyParallelScope.IsChildren [property]: [read-only] bool
 WoofWare.NUnitTestRunner.AssemblyParallelScope.IsFixtures [property]: [read-only] bool
 WoofWare.NUnitTestRunner.AssemblyParallelScope.Tag [property]: [read-only] int
+WoofWare.NUnitTestRunner.BuildTrxReport inherit obj
+WoofWare.NUnitTestRunner.BuildTrxReport.build [static method]: System.Reflection.Assembly -> System.DateTimeOffset -> System.DateTimeOffset -> WoofWare.NUnitTestRunner.FixtureRunResults list -> WoofWare.NUnitTestRunner.TrxReport
 WoofWare.NUnitTestRunner.ClassParallelScope inherit obj, implements WoofWare.NUnitTestRunner.ClassParallelScope System.IEquatable, System.Collections.IStructuralEquatable, WoofWare.NUnitTestRunner.ClassParallelScope System.IComparable, System.IComparable, System.Collections.IStructuralComparable - union type with 4 cases
 WoofWare.NUnitTestRunner.ClassParallelScope+Tags inherit obj
 WoofWare.NUnitTestRunner.ClassParallelScope+Tags.All [static field]: int = 3
@@ -143,6 +168,22 @@ WoofWare.NUnitTestRunner.ITestProgress.OnTestFixtureStart [method]: string -> in
 WoofWare.NUnitTestRunner.ITestProgress.OnTestMemberFinished [method]: string -> unit
 WoofWare.NUnitTestRunner.ITestProgress.OnTestMemberSkipped [method]: string -> unit
 WoofWare.NUnitTestRunner.ITestProgress.OnTestMemberStart [method]: string -> unit
+WoofWare.NUnitTestRunner.LoadContext inherit System.Runtime.Loader.AssemblyLoadContext
+WoofWare.NUnitTestRunner.LoadContext..ctor [constructor]: (System.IO.FileInfo, System.IO.DirectoryInfo list, WoofWare.NUnitTestRunner.TestContexts)
+WoofWare.NUnitTestRunner.LogLevel inherit obj, implements WoofWare.NUnitTestRunner.LogLevel System.IEquatable, System.Collections.IStructuralEquatable, WoofWare.NUnitTestRunner.LogLevel System.IComparable, System.IComparable, System.Collections.IStructuralComparable - union type with 2 cases
+WoofWare.NUnitTestRunner.LogLevel+Tags inherit obj
+WoofWare.NUnitTestRunner.LogLevel+Tags.Nothing [static field]: int = 0
+WoofWare.NUnitTestRunner.LogLevel+Tags.Verbose [static field]: int = 1
+WoofWare.NUnitTestRunner.LogLevel.get_IsNothing [method]: unit -> bool
+WoofWare.NUnitTestRunner.LogLevel.get_IsVerbose [method]: unit -> bool
+WoofWare.NUnitTestRunner.LogLevel.get_Nothing [static method]: unit -> WoofWare.NUnitTestRunner.LogLevel
+WoofWare.NUnitTestRunner.LogLevel.get_Tag [method]: unit -> int
+WoofWare.NUnitTestRunner.LogLevel.get_Verbose [static method]: unit -> WoofWare.NUnitTestRunner.LogLevel
+WoofWare.NUnitTestRunner.LogLevel.IsNothing [property]: [read-only] bool
+WoofWare.NUnitTestRunner.LogLevel.IsVerbose [property]: [read-only] bool
+WoofWare.NUnitTestRunner.LogLevel.Nothing [static property]: [read-only] WoofWare.NUnitTestRunner.LogLevel
+WoofWare.NUnitTestRunner.LogLevel.Tag [property]: [read-only] int
+WoofWare.NUnitTestRunner.LogLevel.Verbose [static property]: [read-only] WoofWare.NUnitTestRunner.LogLevel
 WoofWare.NUnitTestRunner.Match inherit obj, implements WoofWare.NUnitTestRunner.Match System.IEquatable, System.Collections.IStructuralEquatable, WoofWare.NUnitTestRunner.Match System.IComparable, System.IComparable, System.Collections.IStructuralComparable - union type with 2 cases
 WoofWare.NUnitTestRunner.Match+Contains inherit WoofWare.NUnitTestRunner.Match
 WoofWare.NUnitTestRunner.Match+Contains.get_Item [method]: unit -> string
@@ -195,6 +236,15 @@ WoofWare.NUnitTestRunner.Parallelizable`1.IsYes [property]: [read-only] bool
 WoofWare.NUnitTestRunner.Parallelizable`1.NewYes [static method]: 'scope -> 'scope WoofWare.NUnitTestRunner.Parallelizable
 WoofWare.NUnitTestRunner.Parallelizable`1.No [static property]: [read-only] 'scope WoofWare.NUnitTestRunner.Parallelizable
 WoofWare.NUnitTestRunner.Parallelizable`1.Tag [property]: [read-only] int
+WoofWare.NUnitTestRunner.ParallelQueue inherit obj, implements IDisposable
+WoofWare.NUnitTestRunner.ParallelQueue..ctor [constructor]: (int option, WoofWare.NUnitTestRunner.AssemblyParallelScope WoofWare.NUnitTestRunner.Parallelizable option, System.Threading.CancellationToken option)
+WoofWare.NUnitTestRunner.ParallelQueue.EndTestFixture [method]: WoofWare.NUnitTestRunner.TestFixtureTearDownToken -> unit System.Threading.Tasks.Task
+WoofWare.NUnitTestRunner.ParallelQueue.Run [method]: WoofWare.NUnitTestRunner.TestFixtureSetupToken -> unit WoofWare.NUnitTestRunner.Parallelizable option -> (unit -> 'a) -> 'a System.Threading.Tasks.Task
+WoofWare.NUnitTestRunner.ParallelQueue.RunTestSetup [method]: WoofWare.NUnitTestRunner.TestFixtureRunningToken -> (unit -> 'a) -> ('a * WoofWare.NUnitTestRunner.TestFixtureSetupToken) System.Threading.Tasks.Task
+WoofWare.NUnitTestRunner.ParallelQueue.RunTestTearDown [method]: WoofWare.NUnitTestRunner.TestFixtureSetupToken -> (unit -> 'a) -> ('a * WoofWare.NUnitTestRunner.TestFixtureTearDownToken) System.Threading.Tasks.Task
+WoofWare.NUnitTestRunner.ParallelQueue.StartTestFixture [method]: WoofWare.NUnitTestRunner.TestFixture -> WoofWare.NUnitTestRunner.TestFixtureRunningToken System.Threading.Tasks.Task
+WoofWare.NUnitTestRunner.SetBaseDir inherit obj, implements IDisposable
+WoofWare.NUnitTestRunner.SetBaseDir..ctor [constructor]: System.IO.FileInfo
 WoofWare.NUnitTestRunner.SingleTestMethod inherit obj, implements WoofWare.NUnitTestRunner.SingleTestMethod System.IEquatable, System.Collections.IStructuralEquatable
 WoofWare.NUnitTestRunner.SingleTestMethod..ctor [constructor]: (System.Reflection.MethodInfo, WoofWare.NUnitTestRunner.TestKind, WoofWare.NUnitTestRunner.Modifier list, string list, int option, WoofWare.NUnitTestRunner.Combinatorial option, unit WoofWare.NUnitTestRunner.Parallelizable option)
 WoofWare.NUnitTestRunner.SingleTestMethod.Categories [property]: [read-only] string list
@@ -215,6 +265,8 @@ WoofWare.NUnitTestRunner.SingleTestMethod.Parallelize [property]: [read-only] un
 WoofWare.NUnitTestRunner.SingleTestMethod.Repeat [property]: [read-only] int option
 WoofWare.NUnitTestRunner.SingleTestMethodModule inherit obj
 WoofWare.NUnitTestRunner.SingleTestMethodModule.parse [static method]: string list -> System.Reflection.MethodInfo -> System.Reflection.CustomAttributeData list -> (WoofWare.NUnitTestRunner.SingleTestMethod option * System.Reflection.CustomAttributeData list)
+WoofWare.NUnitTestRunner.TestContexts inherit obj, implements WoofWare.NUnitTestRunner.TestContexts System.IEquatable, System.Collections.IStructuralEquatable, IDisposable
+WoofWare.NUnitTestRunner.TestContexts.Empty [static method]: unit -> WoofWare.NUnitTestRunner.TestContexts
 WoofWare.NUnitTestRunner.TestFailure inherit obj, implements WoofWare.NUnitTestRunner.TestFailure System.IEquatable, System.Collections.IStructuralEquatable - union type with 3 cases
 WoofWare.NUnitTestRunner.TestFailure+SetUpFailed inherit WoofWare.NUnitTestRunner.TestFailure
 WoofWare.NUnitTestRunner.TestFailure+SetUpFailed.get_Item [method]: unit -> WoofWare.NUnitTestRunner.UserMethodFailure
@@ -267,7 +319,11 @@ WoofWare.NUnitTestRunner.TestFixture.Tests [property]: [read-only] WoofWare.NUni
 WoofWare.NUnitTestRunner.TestFixture.Type [property]: [read-only] System.Type
 WoofWare.NUnitTestRunner.TestFixtureModule inherit obj
 WoofWare.NUnitTestRunner.TestFixtureModule.parse [static method]: System.Type -> WoofWare.NUnitTestRunner.TestFixture
-WoofWare.NUnitTestRunner.TestFixtureModule.run [static method]: WoofWare.NUnitTestRunner.ITestProgress -> (WoofWare.NUnitTestRunner.TestFixture -> WoofWare.NUnitTestRunner.SingleTestMethod -> bool) -> WoofWare.NUnitTestRunner.TestFixture -> WoofWare.NUnitTestRunner.FixtureRunResults list
+WoofWare.NUnitTestRunner.TestFixtureModule.run [static method]: WoofWare.NUnitTestRunner.TestContexts -> WoofWare.NUnitTestRunner.ParallelQueue -> WoofWare.NUnitTestRunner.ITestProgress -> (WoofWare.NUnitTestRunner.TestFixture -> WoofWare.NUnitTestRunner.SingleTestMethod -> bool) -> WoofWare.NUnitTestRunner.TestFixture -> WoofWare.NUnitTestRunner.FixtureRunResults list System.Threading.Tasks.Task
+WoofWare.NUnitTestRunner.TestFixtureModule.runOneFixture [static method]: WoofWare.NUnitTestRunner.TestContexts -> WoofWare.NUnitTestRunner.ParallelQueue -> WoofWare.NUnitTestRunner.ITestProgress -> (WoofWare.NUnitTestRunner.TestFixture -> WoofWare.NUnitTestRunner.SingleTestMethod -> bool) -> string -> obj -> WoofWare.NUnitTestRunner.TestFixture -> WoofWare.NUnitTestRunner.FixtureRunResults System.Threading.Tasks.Task
+WoofWare.NUnitTestRunner.TestFixtureRunningToken inherit obj, implements WoofWare.NUnitTestRunner.TestFixtureRunningToken System.IEquatable, System.Collections.IStructuralEquatable
+WoofWare.NUnitTestRunner.TestFixtureSetupToken inherit obj, implements WoofWare.NUnitTestRunner.TestFixtureSetupToken System.IEquatable, System.Collections.IStructuralEquatable
+WoofWare.NUnitTestRunner.TestFixtureTearDownToken inherit obj, implements WoofWare.NUnitTestRunner.TestFixtureTearDownToken System.IEquatable, System.Collections.IStructuralEquatable
 WoofWare.NUnitTestRunner.TestKind inherit obj, implements WoofWare.NUnitTestRunner.TestKind System.IEquatable, System.Collections.IStructuralEquatable - union type with 3 cases
 WoofWare.NUnitTestRunner.TestKind+Data inherit WoofWare.NUnitTestRunner.TestKind
 WoofWare.NUnitTestRunner.TestKind+Data.get_Item [method]: unit -> obj list list

WoofWare.NUnitTestRunner.Lib/TestFixture.fs

@@ -6,21 +6,9 @@ open System.Diagnostics
 open System.IO
 open System.Reflection
 open System.Threading
+open System.Threading.Tasks
 open Microsoft.FSharp.Core
 
-type private StdoutSetter (newStdout : StreamWriter, newStderr : StreamWriter) =
-    let oldStdout = Console.Out
-    let oldStderr = Console.Error
-
-    do
-        Console.SetOut newStdout
-        Console.SetError newStderr
-
-    interface IDisposable with
-        member _.Dispose () =
-            Console.SetOut oldStdout
-            Console.SetError oldStderr
-
 /// Information about the circumstances of a run of a single test.
 type IndividualTestRunMetadata =
     {
@@ -80,6 +68,8 @@ module TestFixture =
     ///
     /// This function does not throw.
     let private runOne
+        (outputId : OutputStreamId)
+        (contexts : TestContexts)
         (setUp : MethodInfo list)
         (tearDown : MethodInfo list)
         (testId : Guid)
@@ -112,13 +102,6 @@ module TestFixture =
 
         let start = DateTimeOffset.Now
 
-        use stdOutStream = new MemoryStream ()
-        use stdErrStream = new MemoryStream ()
-        use stdOut = new StreamWriter (stdOutStream)
-        use stdErr = new StreamWriter (stdErrStream)
-
-        use _ = new StdoutSetter (stdOut, stdErr)
-
         let sw = Stopwatch.StartNew ()
 
         let metadata () =
@@ -139,13 +122,13 @@ module TestFixture =
                 TestName = name
                 ClassName = test.DeclaringType.FullName
                 StdOut =
-                    match stdOutStream.ToArray () with
-                    | [||] -> None
-                    | arr -> Console.OutputEncoding.GetString arr |> Some
+                    match contexts.DumpStdout outputId with
+                    | "" -> None
+                    | v -> Some v
                 StdErr =
-                    match stdErrStream.ToArray () with
-                    | [||] -> None
-                    | arr -> Console.OutputEncoding.GetString arr |> Some
+                    match contexts.DumpStderr outputId with
+                    | "" -> None
+                    | v -> Some v
             }
 
         let setUpResult = runMethods TestFailure.SetUpFailed setUp [||]
@@ -223,11 +206,15 @@ module TestFixture =
     /// This method should never throw: it only throws if there's a critical logic error in the runner.
     /// Exceptions from the units under test are wrapped up and passed out.
     let private runTestsFromMember
+        (contexts : TestContexts)
+        (par : ParallelQueue)
+        (running : TestFixtureSetupToken)
+        (progress : ITestProgress)
         (setUp : MethodInfo list)
         (tearDown : MethodInfo list)
         (containingObject : obj)
         (test : SingleTestMethod)
-        : (Result<TestMemberSuccess, TestMemberFailure> * IndividualTestRunMetadata) list
+        : (Result<TestMemberSuccess, TestMemberFailure> * IndividualTestRunMetadata) Task list
         =
         if test.Method.ContainsGenericParameters then
             let failureMetadata =
@@ -247,7 +234,7 @@ module TestFixture =
             let error =
                 TestMemberFailure.Malformed [ "Test contained generic parameters; generics are not supported." ]
 
-            (Error error, failureMetadata) |> List.singleton
+            (Error error, failureMetadata) |> Task.FromResult |> List.singleton
         else
 
         let resultPreRun =
@@ -263,17 +250,12 @@ module TestFixture =
                 | Modifier.Ignored reason -> Some (TestMemberSuccess.Ignored reason)
             )
 
-        let sw = Stopwatch.StartNew ()
-        let startTime = DateTimeOffset.Now
-
         match resultPreRun with
         | Some result ->
-            sw.Stop ()
-
             let failureMetadata =
                 {
-                    Total = sw.Elapsed
-                    Start = startTime
+                    Total = TimeSpan.Zero
+                    Start = DateTimeOffset.Now
                     End = DateTimeOffset.Now
                     ComputerName = Environment.MachineName
                     ExecutionId = Guid.NewGuid ()
@@ -285,7 +267,7 @@ module TestFixture =
                     StdOut = None
                 }
 
-            [ Ok result, failureMetadata ]
+            (Ok result, failureMetadata) |> Task.FromResult |> List.singleton
         | None ->
 
         let individualTests =
@@ -348,7 +330,7 @@ module TestFixture =
 
                             // Might not be an IEnumerable of a reference type.
                             // Concretely, `FSharpList<HttpStatusCode> :> IEnumerable<obj>` fails.
-                            for arg in args.GetValue (null : obj) :?> System.Collections.IEnumerable do
+                            for arg in args.GetValue (null : obj) :?> IEnumerable do
                                 yield
                                     Guid.NewGuid (),
                                     match arg with
@@ -371,20 +353,19 @@ module TestFixture =
                                             if isNull argsMem then
                                                 failwith "Unexpectedly could not call `.Arguments` on TestCaseData"
 
+                                            // TODO: need to capture this stdout/stderr
                                             (argsMem.Invoke (arg, [||]) |> unbox<obj[]>)
                                         else
                                             [| arg |]
                     ]
                     |> Ok
 
-        sw.Stop ()
-
         match individualTests with
         | Error e ->
             let failureMetadata =
                 {
-                    Total = sw.Elapsed
-                    Start = startTime
+                    Total = TimeSpan.Zero
+                    Start = DateTimeOffset.Now
                     End = DateTimeOffset.Now
                     ComputerName = Environment.MachineName
                     ExecutionId = Guid.NewGuid ()
@@ -397,7 +378,7 @@ module TestFixture =
                     StdOut = None
                 }
 
-            [ Error e, failureMetadata ]
+            (Error e, failureMetadata) |> Task.FromResult |> List.singleton
         | Ok individualTests ->
 
         let count = test.Repeat |> Option.defaultValue 1
@@ -405,116 +386,205 @@ module TestFixture =
         Seq.init count (fun _ -> individualTests)
         |> Seq.concat
         |> Seq.map (fun (testGuid, args) ->
-            let results, summary =
-                runOne setUp tearDown testGuid test.Method containingObject args
+            task {
+                let runMe () =
+                    progress.OnTestMemberStart test.Name
+                    let oldValue = contexts.AsyncLocal.Value
+                    let outputId = contexts.NewOutputs ()
+                    contexts.AsyncLocal.Value <- outputId
 
-            match results with
-            | Ok results -> Ok results, summary
-            | Error e -> Error (TestMemberFailure.Failed e), summary
+                    let result, meta =
+                        runOne outputId contexts setUp tearDown testGuid test.Method containingObject args
+
+                    contexts.AsyncLocal.Value <- oldValue
+                    progress.OnTestMemberFinished test.Name
+
+                    result, meta
+
+                let! results, summary = par.Run running test.Parallelize runMe
+
+                match results with
+                | Ok results -> return Ok results, summary
+                | Error e -> return Error (TestMemberFailure.Failed e), summary
+            }
         )
         |> Seq.toList
 
     /// Run every test (except those which fail the `filter`) in this test fixture, as well as the
     /// appropriate setup and tear-down logic.
-    let private runOneFixture
+    let runOneFixture
+        (contexts : TestContexts)
+        (par : ParallelQueue)
         (progress : ITestProgress)
         (filter : TestFixture -> SingleTestMethod -> bool)
         (name : string)
         (containingObject : obj)
         (tests : TestFixture)
-        : FixtureRunResults
+        : FixtureRunResults Task
         =
-        progress.OnTestFixtureStart name tests.Tests.Length
+        task {
+            let! running = par.StartTestFixture tests
+            progress.OnTestFixtureStart name tests.Tests.Length
 
-        let oldWorkDir = Environment.CurrentDirectory
-        Environment.CurrentDirectory <- FileInfo(tests.ContainingAssembly.Location).Directory.FullName
+            let oldWorkDir = Environment.CurrentDirectory
+            Environment.CurrentDirectory <- FileInfo(tests.ContainingAssembly.Location).Directory.FullName
 
-        let sw = Stopwatch.StartNew ()
-        let startTime = DateTimeOffset.UtcNow
+            let sw = Stopwatch.StartNew ()
+            let startTime = DateTimeOffset.UtcNow
 
-        use stdOutStream = new MemoryStream ()
-        use stdOut = new StreamWriter (stdOutStream)
-        use stdErrStream = new MemoryStream ()
-        use stdErr = new StreamWriter (stdErrStream)
-        use _ = new StdoutSetter (stdOut, stdErr)
+            let endMetadata (outputId : OutputStreamId) =
+                let stdOut = contexts.DumpStdout outputId
+                let stdErr = contexts.DumpStderr outputId
 
-        let endMetadata () =
-            let stdOut = stdOutStream.ToArray () |> Console.OutputEncoding.GetString
-            let stdErr = stdErrStream.ToArray () |> Console.OutputEncoding.GetString
+                {
+                    Total = sw.Elapsed
+                    Start = startTime
+                    End = DateTimeOffset.UtcNow
+                    ComputerName = Environment.MachineName
+                    ExecutionId = Guid.NewGuid ()
+                    TestId = Guid.NewGuid ()
+                    // This one is a bit dubious, because we don't actually have a test name at all
+                    TestName = name
+                    ClassName = tests.Name
+                    StdOut = if String.IsNullOrEmpty stdOut then None else Some stdOut
+                    StdErr = if String.IsNullOrEmpty stdErr then None else Some stdErr
+                }
 
-            {
-                Total = sw.Elapsed
-                Start = startTime
-                End = DateTimeOffset.UtcNow
-                ComputerName = Environment.MachineName
-                ExecutionId = Guid.NewGuid ()
-                TestId = Guid.NewGuid ()
-                // This one is a bit dubious, because we don't actually have a test name at all
-                TestName = name
-                ClassName = tests.Name
-                StdOut = if String.IsNullOrEmpty stdOut then None else Some stdOut
-                StdErr = if String.IsNullOrEmpty stdErr then None else Some stdErr
-            }
+            let! setupResult, running =
+                match tests.OneTimeSetUp with
+                | Some su ->
+                    par.RunTestSetup
+                        running
+                        (fun () ->
+                            let oldValue = contexts.AsyncLocal.Value
+                            let newOutputs = contexts.NewOutputs ()
+                            contexts.AsyncLocal.Value <- newOutputs
 
-        let setupResult =
-            match tests.OneTimeSetUp with
-            | Some su ->
-                try
-                    match su.Invoke (containingObject, [||]) with
-                    | :? unit -> None
-                    | ret -> Some (UserMethodFailure.ReturnedNonUnit (su.Name, ret), endMetadata ())
-                with :? TargetInvocationException as e ->
-                    Some (UserMethodFailure.Threw (su.Name, e.InnerException), endMetadata ())
-            | _ -> None
+                            let result =
+                                try
+                                    match su.Invoke (containingObject, [||]) with
+                                    | :? unit -> None
+                                    | ret ->
+                                        Some (UserMethodFailure.ReturnedNonUnit (su.Name, ret), endMetadata newOutputs)
+                                with :? TargetInvocationException as e ->
+                                    Some (UserMethodFailure.Threw (su.Name, e.InnerException), endMetadata newOutputs)
 
-        let testFailures = ResizeArray<TestMemberFailure * IndividualTestRunMetadata> ()
+                            contexts.AsyncLocal.Value <- oldValue
 
-        let successes =
-            ResizeArray<SingleTestMethod * TestMemberSuccess * IndividualTestRunMetadata> ()
+                            match result with
+                            | None -> Ok (Some newOutputs)
+                            | Some err -> Error (err, newOutputs)
+                        )
+                | _ -> Task.FromResult (Ok None, TestFixtureSetupToken.vouchNoSetupRequired running)
 
-        match setupResult with
-        | Some _ ->
-            // Don't run any tests if setup failed.
-            ()
-        | None ->
-            for test in tests.Tests do
-                if filter tests test then
-                    progress.OnTestMemberStart test.Name
-                    let testSuccess = ref 0
+            let testFailures = ResizeArray<TestMemberFailure * IndividualTestRunMetadata> ()
 
-                    let results = runTestsFromMember tests.SetUp tests.TearDown containingObject test
+            let successes =
+                ResizeArray<SingleTestMethod * TestMemberSuccess * IndividualTestRunMetadata> ()
 
-                    for result, report in results do
-                        match result with
-                        | Error failure ->
-                            testFailures.Add (failure, report)
-                            progress.OnTestFailed test.Name failure
-                        | Ok result ->
-                            Interlocked.Increment testSuccess |> ignore<int>
-                            lock successes (fun () -> successes.Add (test, result, report))
+            let testsRun =
+                match setupResult with
+                | Error _ ->
+                    // Don't run any tests if setup failed.
+                    Task.FromResult ()
+                | Ok _ ->
+                    tests.Tests
+                    |> Seq.filter (fun test ->
+                        if filter tests test then
+                            true
+                        else
+                            progress.OnTestMemberSkipped test.Name
+                            false
+                    )
+                    |> Seq.map (fun test ->
+                        task {
+                            let testSuccess = ref 0
 
-                    progress.OnTestMemberFinished test.Name
-                else
-                    progress.OnTestMemberSkipped test.Name
+                            let results =
+                                runTestsFromMember
+                                    contexts
+                                    par
+                                    running
+                                    progress
+                                    tests.SetUp
+                                    tests.TearDown
+                                    containingObject
+                                    test
 
-        // Unconditionally run OneTimeTearDown if it exists.
-        let tearDownError =
-            match tests.OneTimeTearDown with
-            | Some td ->
-                try
-                    match td.Invoke (containingObject, [||]) with
-                    | null -> None
-                    | ret -> Some (UserMethodFailure.ReturnedNonUnit (td.Name, ret), endMetadata ())
-                with :? TargetInvocationException as e ->
-                    Some (UserMethodFailure.Threw (td.Name, e), endMetadata ())
-            | _ -> None
+                            let! result =
+                                results
+                                |> List.map (fun t ->
+                                    task {
+                                        let! result, report = t
 
-        Environment.CurrentDirectory <- oldWorkDir
+                                        match result with
+                                        | Error failure ->
+                                            testFailures.Add (failure, report)
+                                            progress.OnTestFailed test.Name failure
+                                        | Ok result ->
+                                            Interlocked.Increment testSuccess |> ignore<int>
+                                            lock successes (fun () -> successes.Add (test, result, report))
+                                    }
+                                )
+                                |> Task.WhenAll
 
-        {
-            Failed = testFailures |> Seq.toList
-            Success = successes |> Seq.toList
-            OtherFailures = [ tearDownError ; setupResult ] |> List.choose id
+                            result |> Array.iter id
+                        }
+                    )
+                    |> Task.WhenAll
+                    |> fun t ->
+                        task {
+                            let! t = t
+                            return t |> Array.iter id
+                        }
+
+            do! testsRun
+
+            // Unconditionally run OneTimeTearDown if it exists.
+            let! tearDownError, tornDown =
+                match tests.OneTimeTearDown with
+                | Some td ->
+                    par.RunTestTearDown
+                        running
+                        (fun () ->
+                            let oldValue = contexts.AsyncLocal.Value
+                            let outputs = contexts.NewOutputs ()
+                            contexts.AsyncLocal.Value <- outputs
+
+                            let result =
+                                try
+                                    match td.Invoke (containingObject, [||]) with
+                                    | :? unit -> None
+                                    | ret ->
+                                        Some (UserMethodFailure.ReturnedNonUnit (td.Name, ret), endMetadata outputs)
+                                with :? TargetInvocationException as e ->
+                                    Some (UserMethodFailure.Threw (td.Name, e.InnerException), endMetadata outputs)
+
+                            contexts.AsyncLocal.Value <- oldValue
+
+                            match result with
+                            | None -> Ok (Some outputs)
+                            | Some err -> Error (err, outputs)
+                        )
+                | _ -> Task.FromResult (Ok None, TestFixtureTearDownToken.vouchNoTearDownRequired running)
+
+            Environment.CurrentDirectory <- oldWorkDir
+
+            do! par.EndTestFixture tornDown
+
+            // TODO: we have access to stdout/err of OneTimeSetUp and OneTimeTearDown here, but we throw them away.
+            return
+                {
+                    Failed = testFailures |> Seq.toList
+                    Success = successes |> Seq.toList
+                    OtherFailures =
+                        [ tearDownError ; setupResult ]
+                        |> List.choose (
+                            function
+                            | Error (e, _) -> Some e
+                            | Ok _ -> None
+                        )
+                }
         }
 
     /// Interpret this type as a [<TestFixture>], extracting the test members from it and annotating them with all
@@ -639,10 +709,12 @@ module TestFixture =
     /// Run every test (except those which fail the `filter`) in this test fixture, as well as the
     /// appropriate setup and tear-down logic.
     let run
+        (contexts : TestContexts)
+        (par : ParallelQueue)
         (progress : ITestProgress)
         (filter : TestFixture -> SingleTestMethod -> bool)
         (tests : TestFixture)
-        : FixtureRunResults list
+        : FixtureRunResults list Task
         =
         match tests.Parameters with
         | [] -> [ null ]
@@ -678,5 +750,11 @@ module TestFixture =
                     let args = args |> Seq.map (fun o -> o.ToString ()) |> String.concat ","
                     $"%s{tests.Name}(%s{args})"
 
-            runOneFixture progress filter name containingObject tests
+            runOneFixture contexts par progress filter name containingObject tests
         )
+        |> Task.WhenAll
+        |> fun t ->
+            task {
+                let! t = t
+                return Array.toList t
+            }

WoofWare.NUnitTestRunner.Lib/WoofWare.NUnitTestRunner.Lib.fsproj

@@ -14,11 +14,12 @@
       <PackageId>WoofWare.NUnitTestRunner.Lib</PackageId>
       <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
       <WarnOn>FS3559</WarnOn>
-      <WoofWareMyriadPluginVersion>2.1.44</WoofWareMyriadPluginVersion>
+      <WoofWareMyriadPluginVersion>2.1.45</WoofWareMyriadPluginVersion>
     </PropertyGroup>
 
     <ItemGroup>
       <Compile Include="AssemblyInfo.fs" />
+      <Compile Include="AppContext.fs" />
       <Compile Include="RuntimeConfig.fs" />
       <Compile Include="GeneratedRuntimeConfig.fs">
         <MyriadFile>RuntimeConfig.fs</MyriadFile>
@@ -29,11 +30,16 @@
       <Compile Include="Result.fs" />
       <Compile Include="Domain.fs" />
       <Compile Include="Filter.fs" />
+      <Compile Include="Args.fs" />
+      <Compile Include="ParallelQueue.fs" />
       <Compile Include="SingleTestMethod.fs" />
       <Compile Include="TestProgress.fs" />
+      <Compile Include="Context.fs" />
       <Compile Include="TestFixture.fs" />
       <Compile Include="Xml.fs" />
       <Compile Include="TrxReport.fs" />
+      <Compile Include="CreateTrxReport.fs" />
+      <Compile Include="AssemblyLevelAttributes.fs" />
       <None Include="..\README.md">
         <Pack>True</Pack>
         <PackagePath>\</PackagePath>
@@ -42,14 +48,14 @@
       <EmbeddedResource Include="version.json" />
     </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="WoofWare.PrattParser" Version="0.1.2" />
+    <PackageReference Include="WoofWare.PrattParser" Version="0.2.2" />
     <PackageReference Update="FSharp.Core" Version="6.0.0" />
-    <PackageReference Include="WoofWare.DotnetRuntimeLocator" Version="0.1.4" />
-    <PackageReference Include="WoofWare.Myriad.Plugins.Attributes" Version="3.1.6" />
-    <PackageReference Include="Myriad.SDK" Version="0.8.3" />
+    <PackageReference Include="WoofWare.DotnetRuntimeLocator" Version="0.1.9" />
+    <PackageReference Include="WoofWare.Myriad.Plugins.Attributes" Version="3.1.7" />
+    <PackageReference Include="Myriad.SDK" Version="0.8.3" PrivateAssets="all" />
     <PackageReference Include="WoofWare.Myriad.Plugins" Version="$(WoofWareMyriadPluginVersion)" PrivateAssets="all" />
   </ItemGroup>
-  
+
   <ItemGroup>
     <MyriadSdkGenerator Include="$(NuGetPackageRoot)/woofware.myriad.plugins/$(WoofWareMyriadPluginVersion)/lib/net6.0/WoofWare.Myriad.Plugins.dll" />
   </ItemGroup>

WoofWare.NUnitTestRunner.Lib/version.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.12",
+  "version": "0.15",
   "publicReleaseRefSpec": [
     "^refs/heads/main$"
   ],
@@ -8,4 +8,4 @@
     ":/Directory.Build.props",
     ":/README.md"
   ]
-}
+}

WoofWare.NUnitTestRunner/Program.fs

@@ -2,345 +2,82 @@
 
 open System
 open System.IO
-open System.Reflection
-open System.Runtime.Loader
-
-// Fix for https://github.com/Smaug123/unofficial-nunit-runner/issues/8
-// Set AppContext.BaseDirectory to where the test DLL is.
-// (This tells the DLL loader to look next to the test DLL for dependencies.)
-type SetBaseDir (testDll : FileInfo) =
-    let oldBaseDir = AppContext.BaseDirectory
-    do AppContext.SetData ("APP_CONTEXT_BASE_DIRECTORY", testDll.Directory.FullName)
-
-    interface IDisposable with
-        member _.Dispose () =
-            AppContext.SetData ("APP_CONTEXT_BASE_DIRECTORY", oldBaseDir)
-
-
-type Ctx (dll : FileInfo, runtimes : DirectoryInfo list) =
-    inherit AssemblyLoadContext ()
-
-    override this.Load (target : AssemblyName) : Assembly =
-        let path = Path.Combine (dll.Directory.FullName, $"%s{target.Name}.dll")
-
-        if File.Exists path then
-            this.LoadFromAssemblyPath path
-        else
-
-        runtimes
-        |> List.tryPick (fun di ->
-            let path = Path.Combine (di.FullName, $"%s{target.Name}.dll")
-
-            if File.Exists path then
-                this.LoadFromAssemblyPath path |> Some
-            else
-                None
-        )
-        |> Option.defaultValue null
-
+open System.Threading.Tasks
+open Spectre.Console
 
 module Program =
     let main argv =
         let startTime = DateTimeOffset.Now
 
-        let testDll, filter, trxPath =
-            match argv |> List.ofSeq with
-            | [ dll ] -> FileInfo dll, None, None
-            | [ dll ; "--trx" ; trxPath ] -> FileInfo dll, None, Some (FileInfo trxPath)
-            | [ dll ; "--filter" ; filter ] -> FileInfo dll, Some (Filter.parse filter), None
-            | [ dll ; "--trx" ; trxPath ; "--filter" ; filter ] ->
-                FileInfo dll, Some (Filter.parse filter), Some (FileInfo trxPath)
-            | [ dll ; "--filter" ; filter ; "--trx" ; trxPath ] ->
-                FileInfo dll, Some (Filter.parse filter), Some (FileInfo trxPath)
-            | _ ->
-                failwith
-                    "provide exactly one arg, a test DLL; then optionally `--filter <filter>` and/or `--trx <output-filename>`."
+        let args = argv |> List.ofArray |> Args.Parse
 
         let filter =
-            match filter with
-            | Some filter -> Filter.shouldRun filter
+            match args.Filter with
+            | Some (_, filter) -> Filter.shouldRun filter
             | None -> fun _ _ -> true
 
-        let progress = Progress.spectre ()
+        let stderr =
+            let consoleSettings = AnsiConsoleSettings ()
+            consoleSettings.Out <- AnsiConsoleOutput Console.Error
+            AnsiConsole.Create consoleSettings
 
-        use _ = new SetBaseDir (testDll)
+        let progress = Progress.spectre stderr
 
-        let ctx = Ctx (testDll, DotnetRuntime.locate testDll)
-        let assy = ctx.LoadFromAssemblyPath testDll.FullName
+        let runtime = DotnetRuntime.locate args.Dll
 
-        let levelOfParallelism, par =
-            ((None, None), assy.CustomAttributes)
-            ||> Seq.fold (fun (levelPar, par) attr ->
-                match attr.AttributeType.FullName with
-                | "NUnit.Framework.LevelOfParallelismAttribute" ->
-                    let arg = attr.ConstructorArguments |> Seq.exactlyOne |> _.Value |> unbox<int>
+        match args.Logging with
+        | LogLevel.Nothing -> ()
+        | LogLevel.Verbose ->
+            for d in runtime do
+                stderr.WriteLine $".NET runtime directory: %s{d.FullName}"
 
-                    match levelPar with
-                    | None -> (Some arg, par)
-                    | Some existing ->
-                        failwith $"Assembly %s{assy.Location} declares parallelism %i{arg} and also %i{existing}"
-                | "NUnit.Framework.NonParallelizableAttribute" ->
-                    match levelPar with
-                    | None -> (Some 1, par)
-                    | Some existing ->
-                        failwith
-                            $"Assembly %s{assy.Location} declares non-parallelizable and also parallelism %i{existing}"
-                | "NUnit.Framework.ParallelizableAttribute" ->
-                    match par with
-                    | Some _ -> failwith "Got multiple Parallelize attributes in assembly"
-                    | None ->
-                        match attr.ConstructorArguments |> Seq.toList with
-                        | [] -> levelPar, Some (Parallelizable.Yes AssemblyParallelScope.Fixtures)
-                        | [ v ] ->
-                            match v.Value with
-                            | :? int as v ->
-                                match v with
-                                | 512 -> levelPar, Some (Parallelizable.Yes AssemblyParallelScope.Fixtures)
-                                | 256 -> levelPar, Some (Parallelizable.Yes AssemblyParallelScope.Children)
-                                | 257 ->
-                                    failwith "ParallelScope.All is invalid on assemblies; only Fixtures or Children"
-                                | 1 ->
-                                    failwith "ParallelScope.Self is invalid on assemblies; only Fixtures or Children"
-                                | v -> failwith $"Could not recognise value %i{v} of parallel scope on assembly"
-                            | v -> failwith $"Unexpectedly non-int value %O{v} of parallel scope on assembly"
-                        | _ -> failwith "unexpectedly got multiple args to Parallelizable on assembly"
-                | _ -> levelPar, par
-            )
+        use _ = new SetBaseDir (args.Dll)
+
+        use contexts = TestContexts.Empty ()
+
+        let ctx = LoadContext (args.Dll, runtime, contexts)
+        let assy = ctx.LoadFromAssemblyPath args.Dll.FullName
+
+        let attrs = AssemblyLevelAttributes.get assy
+
+        let levelOfParallelism =
+            match args.LevelOfParallelism, attrs.Parallelism with
+            | None, None -> None
+            | Some taken, Some ignored ->
+                match args.Logging with
+                | LogLevel.Nothing -> ()
+                | LogLevel.Verbose ->
+                    stderr.WriteLine
+                        $"Taking parallelism %i{taken} from command line, ignoring value %i{ignored} from assembly"
+
+                Some taken
+            | Some x, None
+            | None, Some x -> Some x
 
         let testFixtures = assy.ExportedTypes |> Seq.map TestFixture.parse |> Seq.toList
 
+        use par = new ParallelQueue (levelOfParallelism, attrs.Parallelizable)
+
         let creationTime = DateTimeOffset.Now
-        let results = testFixtures |> List.collect (TestFixture.run progress filter)
-
-        let finishTime = DateTimeOffset.Now
-        let finishTimeHumanReadable = finishTime.ToString @"yyyy-MM-dd HH:mm:ss"
-        let nowMachine = finishTime.ToString @"yyyy-MM-dd_HH_mm_ss"
-
-        let testListId = Guid.NewGuid ()
-
-        let testDefinitions, testEntries =
-            results
-            |> List.collect (fun results -> results.IndividualTestRunMetadata)
-            |> List.map (fun (data, _) ->
-                let defn =
-                    {
-                        Name = data.TestName
-                        Storage = assy.Location.ToLowerInvariant ()
-                        Id = data.TestId
-                        Execution =
-                            {
-                                Id = data.ExecutionId
-                            }
-                        TestMethod =
-                            {
-                                CodeBase = assy.Location
-                                AdapterTypeName = Uri "executor://woofware/"
-                                ClassName = data.ClassName
-                                Name = data.TestName
-                            }
-                    }
-
-                let entry : TrxTestEntry =
-                    {
-                        TestListId = testListId
-                        ExecutionId = data.ExecutionId
-                        TestId = data.TestId
-
-                    }
-
-                defn, entry
-            )
-            |> List.unzip
-
-        let hostname = Environment.MachineName
-
-        let settings =
-            {
-                Name = "default"
-                Id = Guid.NewGuid ()
-                Deployment =
-                    {
-                        RunDeploymentRoot = $"_%s{hostname}_%s{nowMachine}"
-                    }
-            }
-
-        let testList : TrxTestListEntry =
-            {
-                Id = testListId
-                Name = "All"
-            }
-
-        let counters =
-            (TrxCounters.Zero, results)
-            // TODO: this is woefully inefficient
-            ||> List.fold (fun counters results ->
-                let counters =
-                    (counters, results.Failed)
-                    ||> List.fold (fun counters (_, _) ->
-                        // TODO: the counters can be more specific about the failure mode
-                        counters.AddFailed ()
-                    )
-
-                let counters =
-                    (counters, results.OtherFailures)
-                    ||> List.fold (fun counters _ ->
-                        // TODO: the counters can be more specific about the failure mode
-                        counters.AddFailed ()
-                    )
-
-                (counters, results.Success)
-                ||> List.fold (fun counters (_, success, _) ->
-                    match success with
-                    | TestMemberSuccess.Ok -> counters.AddPassed ()
-                    | TestMemberSuccess.Ignored _
-                    | TestMemberSuccess.Explicit _ -> counters.AddNotExecuted ()
-                    | TestMemberSuccess.Inconclusive _ -> counters.AddInconclusive ()
-                )
-            )
-
-        // TODO: I'm sure we can do better than this; there's a whole range of possible
-        // states!
-        let outcome =
-            if counters.Failed > 0u then
-                TrxOutcome.Failed
-            else
-                TrxOutcome.Completed
-
-        let resultSummary : TrxResultsSummary =
-            {
-                Outcome = outcome
-                Counters = counters
-                Output =
-                    {
-                        StdOut = None
-                        StdErr = None
-                        ErrorInfo = None
-                    }
-                RunInfos =
-                    [
-                    // TODO: capture stdout
-                    ]
-            }
-
-        let times : TrxReportTimes =
-            {
-                Creation = creationTime
-                Queuing = startTime
-                Start = startTime
-                Finish = finishTime
-
-            }
-
-        let magicGuid = Guid.Parse "13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b"
 
         let results =
-            results
-            |> List.collect (fun results -> results.IndividualTestRunMetadata)
-            |> List.map (fun (i, cause) ->
-                let exc =
-                    match cause with
-                    | Choice2Of3 _ -> None
-                    | Choice1Of3 (TestMemberFailure.Malformed reasons) ->
-                        {
-                            StackTrace = None
-                            Message = reasons |> String.concat "\n" |> Some
-                        }
-                        |> Some
-                    | Choice1Of3 (TestMemberFailure.Failed fail)
-                    | Choice1Of3 (TestMemberFailure.Failed fail)
-                    | Choice1Of3 (TestMemberFailure.Failed fail) ->
-                        ((None, None), fail)
-                        ||> List.fold (fun (stackTrace, message) tf ->
-                            match tf with
-                            | TestFailure.TestFailed (UserMethodFailure.Threw (_, exc))
-                            | TestFailure.SetUpFailed (UserMethodFailure.Threw (_, exc))
-                            | TestFailure.TearDownFailed (UserMethodFailure.Threw (_, exc)) ->
-                                let stackTrace =
-                                    match stackTrace with
-                                    | None -> (exc : Exception).ToString ()
-                                    | Some s -> s
+            testFixtures
+            |> List.map (TestFixture.run contexts par progress filter)
+            |> Task.WhenAll
 
-                                (Some stackTrace, message)
-                            | TestFailure.TestFailed (UserMethodFailure.ReturnedNonUnit (_, ret))
-                            | TestFailure.SetUpFailed (UserMethodFailure.ReturnedNonUnit (_, ret))
-                            | TestFailure.TearDownFailed (UserMethodFailure.ReturnedNonUnit (_, ret)) ->
-                                let newMessage = $"returned non-unit value %O{ret}"
+        let timeout =
+            match args.Timeout with
+            | None -> TimeSpan.FromHours 2.0
+            | Some t -> t
 
-                                let message =
-                                    match message with
-                                    | None -> newMessage
-                                    | Some message -> $"%s{message}\n%s{newMessage}"
+        if not (results.Wait timeout) then
+            failwith "Tests failed to terminate within two hours"
 
-                                (stackTrace, Some message)
-                        )
-                        |> fun (stackTrace, message) ->
-                            {
-                                StackTrace = stackTrace
-                                Message = message
-                            }
-                            |> Some
-                    | Choice3Of3 (UserMethodFailure.Threw (_, exc)) ->
-                        {
-                            StackTrace = (exc : Exception).ToString () |> Some
-                            Message = None
-                        }
-                        |> Some
-                    | Choice3Of3 (UserMethodFailure.ReturnedNonUnit (_, ret)) ->
-                        {
-                            Message = $"returned non-unit value %O{ret}" |> Some
-                            StackTrace = None
-                        }
-                        |> Some
+        let results = results.Result |> Seq.concat |> List.ofSeq
 
-                let outcome =
-                    match cause with
-                    | Choice1Of3 _ -> TrxTestOutcome.Failed
-                    | Choice2Of3 TestMemberSuccess.Ok -> TrxTestOutcome.Passed
-                    | Choice2Of3 (TestMemberSuccess.Inconclusive _) -> TrxTestOutcome.Inconclusive
-                    | Choice2Of3 (TestMemberSuccess.Ignored _)
-                    | Choice2Of3 (TestMemberSuccess.Explicit _) -> TrxTestOutcome.NotExecuted
-                    // TODO: we can totally do better here, more fine-grained classification
-                    | Choice3Of3 _ -> TrxTestOutcome.Failed
+        let report = BuildTrxReport.build assy creationTime startTime results
 
-                {
-                    ExecutionId = i.ExecutionId
-                    TestId = i.TestId
-                    TestName = i.TestName
-                    ComputerName = i.ComputerName
-                    Duration = i.End - i.Start
-                    StartTime = i.Start
-                    EndTime = i.End
-                    TestType = magicGuid
-                    Outcome = outcome
-                    TestListId = testListId
-                    RelativeResultsDirectory = i.ExecutionId.ToString () // for some reason
-                    Output =
-                        match i.StdOut, i.StdErr, exc with
-                        | None, None, None -> None
-                        | stdout, stderr, exc ->
-                            Some
-                                {
-                                    TrxOutput.StdOut = stdout
-                                    StdErr = stderr
-                                    ErrorInfo = exc
-                                }
-                }
-            )
-
-        let report : TrxReport =
-            {
-                Id = Guid.NewGuid ()
-                Name = $"@%s{hostname} %s{finishTimeHumanReadable}"
-                Times = times
-                Settings = settings
-                Results = results
-                TestDefinitions = testDefinitions
-                TestEntries = testEntries
-                TestLists = [ testList ]
-                ResultsSummary = resultSummary
-            }
-
-        match trxPath with
+        match args.Trx with
         | Some trxPath ->
             let contents = TrxReport.toXml report |> fun d -> d.OuterXml
             trxPath.Directory.Create ()
@@ -348,7 +85,7 @@ module Program =
             Console.Error.WriteLine $"Written TRX file: %s{trxPath.FullName}"
         | None -> ()
 
-        match outcome with
+        match report.ResultsSummary.Outcome with
         | TrxOutcome.Completed -> 0
         | _ -> 1
 

WoofWare.NUnitTestRunner/Progress.fs

@@ -4,21 +4,21 @@ open Spectre.Console
 
 [<RequireQualifiedAccess>]
 module Progress =
-    let spectre () : ITestProgress =
+    let spectre (console : IAnsiConsole) : ITestProgress =
         { new ITestProgress with
             member _.OnTestFailed name failure =
-                AnsiConsole.Console.MarkupLine
+                console.MarkupLine
                     $"[red]Test '%s{Markup.Escape name}' failed: %s{Markup.Escape (failure.ToString ())}[/]"
 
             member _.OnTestFixtureStart name testCount =
-                AnsiConsole.Console.MarkupLine $"[white]Running tests: %s{Markup.Escape name}[/]"
+                console.MarkupLine $"[white]Running tests: %s{Markup.Escape name}[/]"
 
             member _.OnTestMemberFinished name =
-                AnsiConsole.Console.MarkupLine $"[gray]Finished test: %s{Markup.Escape name}[/]"
+                console.MarkupLine $"[gray]Finished test: %s{Markup.Escape name}[/]"
 
             member _.OnTestMemberSkipped name =
-                AnsiConsole.Console.MarkupLine $"[yellow]Skipping test due to filter: %s{Markup.Escape name}[/]"
+                console.MarkupLine $"[yellow]Skipping test due to filter: %s{Markup.Escape name}[/]"
 
             member _.OnTestMemberStart name =
-                AnsiConsole.Console.MarkupLine $"[white]Running test: %s{Markup.Escape name}[/]"
+                console.MarkupLine $"[white]Running test: %s{Markup.Escape name}[/]"
         }

WoofWare.NUnitTestRunner/Seq.fs

@@ -1,22 +0,0 @@
-namespace WoofWare.NUnitTestRunner
-
-[<RequireQualifiedAccess>]
-module internal Seq =
-
-    let tryMinBy (f : 'a -> 'b) (s : 'a seq) : 'a option =
-        use enum = s.GetEnumerator ()
-
-        if not (enum.MoveNext ()) then
-            None
-        else
-
-        let mutable answer = enum.Current
-        let mutable fAnswer = f enum.Current
-
-        while enum.MoveNext () do
-            let fNext = f enum.Current
-
-            if fNext < fAnswer then
-                answer <- enum.Current
-
-        Some answer

WoofWare.NUnitTestRunner/WoofWare.NUnitTestRunner.fsproj

@@ -19,7 +19,6 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <Compile Include="Seq.fs" />
     <Compile Include="Progress.fs" />
     <Compile Include="Program.fs" />
     <None Include="..\README.md">

flake.nix

@@ -57,8 +57,8 @@
         };
       };
       devShell = pkgs.mkShell {
-        buildInputs = [dotnet-sdk];
         packages = [
+          (pkgs.dotnetCorePackages.combinePackages [dotnet-sdk pkgs.dotnetCorePackages.runtime_6_0])
           pkgs.alejandra
           pkgs.nodePackages.markdown-link-check
           pkgs.shellcheck

nix/deps.nix

@@ -248,22 +248,22 @@
   })
   (fetchNuGet {
     pname = "WoofWare.DotnetRuntimeLocator";
-    version = "0.1.4";
-    sha256 = "19pp4qlyf18g704ppbcsm1rhjqjpi84py18yljj9nx70331m8bpg";
+    version = "0.1.9";
+    sha256 = "14yc3ixcn58wy0v3pbj0hjfj4iv5k1ckig0dg1n7njx30510kzyj";
   })
   (fetchNuGet {
     pname = "WoofWare.Myriad.Plugins";
-    version = "2.1.44";
-    sha256 = "0rp9hpkah60gd9x0ba2izr9ff1g7yhzv5a4pkhi5fbrwf5rpqpwx";
+    version = "2.1.45";
+    sha256 = "1i9s9aq8dqnxyn01sa10dd24y9i7cgv2d0rshmrkvbvbjkcnz9vs";
   })
   (fetchNuGet {
     pname = "WoofWare.Myriad.Plugins.Attributes";
-    version = "3.1.6";
-    sha256 = "0786pr1p0nq0854mqi2cddmh185j3jihwn6azz9wiy6nxawjbrd2";
+    version = "3.1.7";
+    sha256 = "1v1wsrjh7qz2khrlbcysj50yydqc9njj09vs1jglwscjhml1wl1v";
   })
   (fetchNuGet {
     pname = "WoofWare.PrattParser";
-    version = "0.1.2";
-    sha256 = "0spypcwsbn805yrs6grjj68ccva902lhkq93mxy32rdply1xs34q";
+    version = "0.2.2";
+    sha256 = "0cgrmd1kc3k224lsjhy5npalwg6kpqd8nx78szi1yq67kyb0farq";
   })
 ]