diff --git a/.github/workflows/dotnet.yaml b/.github/workflows/dotnet.yaml index 5cf4de0..8630ef6 100644 --- a/.github/workflows/dotnet.yaml +++ b/.github/workflows/dotnet.yaml @@ -36,9 +36,34 @@ jobs: - name: Restore dependencies run: nix develop --command dotnet restore - name: Build - run: nix develop --command dotnet build --no-restore --configuration ${{matrix.config}} + run: 'nix develop --command dotnet build --no-restore --configuration ${{matrix.config}}' - name: Test - run: nix develop --command dotnet test --no-build --verbosity normal --configuration ${{matrix.config}} + run: 'nix develop --command dotnet test --no-build --verbosity normal --configuration ${{matrix.config}}' + + selftest: + strategy: + matrix: + config: + - Release + - Debug + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # so that NerdBank.GitVersioning has access to history + - name: Install Nix + uses: cachix/install-nix-action@V27 + with: + extra_nix_config: | + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + - name: Restore dependencies + run: nix develop --command dotnet restore + - name: Build + run: 'nix develop --command dotnet build --no-restore --configuration ${{matrix.config}}' + - name: Test using self + run: 'nix develop --command dotnet exec ./TestRunner/bin/${{matrix.config}}/net8.0/TestRunner.dll ./Consumer/bin/${{matrix.config}}/net8.0/Consumer.dll' analyzers: runs-on: ubuntu-latest diff --git a/Consumer/Consumer.fsproj b/Consumer/Consumer.fsproj new file mode 100644 index 0000000..fa1eddf --- /dev/null +++ b/Consumer/Consumer.fsproj @@ -0,0 +1,21 @@ + + + + net8.0 + + false + true + + + + + + + + + + + + + + diff --git a/Consumer/TestSetUp.fs b/Consumer/TestSetUp.fs new file mode 100644 index 0000000..6e09940 --- /dev/null +++ b/Consumer/TestSetUp.fs @@ -0,0 +1,61 @@ +namespace Consumer + +open FsUnitTyped +open System.Threading +open NUnit.Framework + +[] +module TestSetUp = + + let haveOneTimeSetUp = ref 0 + + [] + let oneTimeSetUp () = + if Interlocked.Increment haveOneTimeSetUp <> 1 then + failwith "one time setup happened more than once" + + let setUpTimes = ref 0 + let tearDownTimes = ref 0 + + let setUpTimesSeen = ResizeArray () + let tearDownTimesSeen = ResizeArray () + + [] + let setUp () = + haveOneTimeSetUp.Value |> shouldEqual 1 + Interlocked.Increment setUpTimes |> setUpTimesSeen.Add + + [] + let tearDown () = + Interlocked.Increment tearDownTimes |> tearDownTimesSeen.Add + + let haveOneTimeTearDown = ref 0 + + [] + let oneTimeTearDown () = + if Interlocked.Increment haveOneTimeTearDown <> 1 then + failwith "one time tear down happened more than once" + + setUpTimesSeen + |> Seq.toList + // Six tests: one for Test, two for the TestCase, three for the Repeat. + |> shouldEqual [ 1..6 ] + + tearDownTimesSeen |> Seq.toList |> shouldEqual [ 1..6 ] + + [] + let ``Test 1`` () = + haveOneTimeTearDown.Value |> shouldEqual 0 + 1 |> shouldEqual 1 + + [] + [] + let ``Test 2`` (s : string) = + haveOneTimeTearDown.Value |> shouldEqual 0 + s.Length |> shouldEqual 1 + + [] + [] + let ``Test 3`` () = + haveOneTimeTearDown.Value |> shouldEqual 0 + 1 |> shouldEqual 1 diff --git a/TestRunner.sln b/TestRunner.sln index eff2278..9d38ebd 100644 --- a/TestRunner.sln +++ b/TestRunner.sln @@ -4,6 +4,8 @@ Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "TestRunner", "TestRunner\Te EndProject Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "TestRunner.Test", "TestRunner\TestRunner.Test\TestRunner.Test.fsproj", "{E776AC80-CD07-4A3E-9F85-1AEFBB56309D}" EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Consumer", "Consumer\Consumer.fsproj", "{5C87D399-62EB-4A5F-8F6C-3FD6F1B31684}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -18,5 +20,9 @@ Global {E776AC80-CD07-4A3E-9F85-1AEFBB56309D}.Debug|Any CPU.Build.0 = Debug|Any CPU {E776AC80-CD07-4A3E-9F85-1AEFBB56309D}.Release|Any CPU.ActiveCfg = Release|Any CPU {E776AC80-CD07-4A3E-9F85-1AEFBB56309D}.Release|Any CPU.Build.0 = Release|Any CPU + {5C87D399-62EB-4A5F-8F6C-3FD6F1B31684}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5C87D399-62EB-4A5F-8F6C-3FD6F1B31684}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5C87D399-62EB-4A5F-8F6C-3FD6F1B31684}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5C87D399-62EB-4A5F-8F6C-3FD6F1B31684}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/TestRunner/Program.fs b/TestRunner/Program.fs index b14cf03..bd78d66 100644 --- a/TestRunner/Program.fs +++ b/TestRunner/Program.fs @@ -1,6 +1,7 @@ namespace TestRunner open System +open System.Collections.Generic open System.IO open System.Reflection open System.Threading @@ -124,6 +125,8 @@ type TestFixture = Name : string OneTimeSetUp : MethodInfo option OneTimeTearDown : MethodInfo option + SetUp : MethodInfo list + TearDown : MethodInfo list Tests : SingleTestMethod list } @@ -132,6 +135,8 @@ type TestFixture = Name = name OneTimeSetUp = None OneTimeTearDown = None + SetUp = [] + TearDown = [] Tests = [] } @@ -146,15 +151,36 @@ type TestFailure = [] module TestFixture = - let private runOne (test : MethodInfo) (args : obj[]) : Result = + let private runOne + (setUp : MethodInfo list) + (tearDown : MethodInfo list) + (test : MethodInfo) + (args : obj[]) + : Result + = try - match test.Invoke (null, args) with - | :? unit -> Ok () - | ret -> Error (TestReturnedNonUnit ret) - with exc -> - Error (TestThrew exc) + for setup in setUp do + if not (isNull (setup.Invoke (null, [||]))) then + failwith $"Setup procedure '%s{setup.Name}' returned non-null" - let private runFixture (test : SingleTestMethod) : Result list = + try + match test.Invoke (null, args) with + | :? unit -> Ok () + | ret -> Error (TestReturnedNonUnit ret) + with exc -> + Error (TestThrew exc) + + finally + for tearDown in tearDown do + if not (isNull (tearDown.Invoke (null, [||]))) then + failwith $"Teardown procedure '%s{tearDown.Name}' returned non-null" + + let private runFixture + (setUp : MethodInfo list) + (tearDown : MethodInfo list) + (test : SingleTestMethod) + : Result list + = let shouldRunTest = (true, test.Modifiers) ||> List.fold (fun _ modifier -> @@ -186,8 +212,10 @@ module TestFixture = (Option.defaultValue 1 test.Repeat) (fun _ -> match test.Kind with - | TestKind.Data data -> data |> Seq.map (fun args -> runOne test.Method (Array.ofList args)) - | TestKind.Single -> Seq.singleton (runOne test.Method [||]) + | TestKind.Data data -> + data + |> Seq.map (fun args -> runOne setUp tearDown test.Method (Array.ofList args)) + | TestKind.Single -> Seq.singleton (runOne setUp tearDown test.Method [||]) | TestKind.Source s -> let args = test.Method.DeclaringType.GetProperty ( @@ -204,12 +232,13 @@ module TestFixture = for arg in args.GetValue null :?> System.Collections.IEnumerable do yield match arg with - | :? TestCaseData as tcd -> runOne test.Method tcd.Arguments - | :? Tuple as (a, b) -> runOne test.Method [| a ; b |] - | :? Tuple as (a, b, c) -> runOne test.Method [| a ; b ; c |] + | :? TestCaseData as tcd -> runOne setUp tearDown test.Method tcd.Arguments + | :? Tuple as (a, b) -> runOne setUp tearDown test.Method [| a ; b |] + | :? Tuple as (a, b, c) -> + runOne setUp tearDown test.Method [| a ; b ; c |] | :? Tuple as (a, b, c, d) -> - runOne test.Method [| a ; b ; c ; d |] - | arg -> runOne test.Method [| arg |] + runOne setUp tearDown test.Method [| a ; b ; c ; d |] + | arg -> runOne setUp tearDown test.Method [| arg |] } ) |> Seq.concat @@ -243,7 +272,7 @@ module TestFixture = match tests.OneTimeSetUp with | Some su -> if not (isNull (su.Invoke (null, [||]))) then - failwith "Setup procedure returned non-null" + failwith $"One-time setup procedure '%s{su.Name}' returned non-null" | _ -> () let totalTestSuccess = ref 0 @@ -255,7 +284,7 @@ module TestFixture = eprintfn $"Running test: %s{test.Name}" let testSuccess = ref 0 - let results = runFixture test + let results = runFixture tests.SetUp tests.TearDown test for result in results do match result with @@ -272,7 +301,7 @@ module TestFixture = match tests.OneTimeTearDown with | Some td -> if not (isNull (td.Invoke (null, [||]))) then - failwith "TearDown procedure returned non-null" + failwith $"TearDown procedure '%s{td.Name}' returned non-null" | _ -> () eprintfn $"Test fixture %s{tests.Name} completed (%i{totalTestSuccess.Value} success)." @@ -307,6 +336,20 @@ module TestFixture = OneTimeTearDown = Some mi } | Some _existing -> failwith "Multiple OneTimeTearDown methods found" + elif + mi.CustomAttributes + |> Seq.exists (fun attr -> attr.AttributeType.FullName = "NUnit.Framework.TearDownAttribute") + then + { state with + TearDown = mi :: state.TearDown + } + elif + mi.CustomAttributes + |> Seq.exists (fun attr -> attr.AttributeType.FullName = "NUnit.Framework.SetUpAttribute") + then + { state with + SetUp = mi :: state.SetUp + } else match SingleTestMethod.parse categories mi with | Some test ->