From ace1417de6f93f6feb775dd1e6e5ae8e04fc65bc Mon Sep 17 00:00:00 2001 From: Patrick Stevens <3138005+Smaug123@users.noreply.github.com> Date: Wed, 30 Oct 2024 19:33:28 +0000 Subject: [PATCH] Hang when argument not supplied (#171) --- .github/workflows/dotnet.yaml | 25 ++++++++++++++++++- FailingConsumer/FailingConsumer.fsproj | 20 +++++++++++++++ FailingConsumer/TestInsufficientArgs.fs | 9 +++++++ FailingConsumer/expected.txt | 1 + .../CreateTrxReport.fs | 20 +++++++++++++++ WoofWare.NUnitTestRunner.Lib/Domain.fs | 12 +++++++++ WoofWare.NUnitTestRunner.Lib/ParallelQueue.fs | 2 +- .../SurfaceBaseline.txt | 13 +++++++++- WoofWare.NUnitTestRunner.Lib/TestFixture.fs | 11 ++++++-- WoofWare.NUnitTestRunner.Lib/version.json | 4 +-- WoofWare.NUnitTestRunner.sln | 6 +++++ flake.nix | 1 + 12 files changed, 117 insertions(+), 7 deletions(-) create mode 100644 FailingConsumer/FailingConsumer.fsproj create mode 100644 FailingConsumer/TestInsufficientArgs.fs create mode 100644 FailingConsumer/expected.txt diff --git a/.github/workflows/dotnet.yaml b/.github/workflows/dotnet.yaml index bd0b300..c6f40d8 100644 --- a/.github/workflows/dotnet.yaml +++ b/.github/workflows/dotnet.yaml @@ -38,7 +38,30 @@ jobs: - name: Build run: 'nix develop --command dotnet build --no-restore --configuration ${{matrix.config}}' - name: Test - run: 'nix develop --command dotnet test --no-build --verbosity normal --configuration ${{matrix.config}} --framework net8.0' + run: | + nix develop --command dotnet test --no-build --verbosity normal --configuration ${{matrix.config}} --framework net8.0 --filter 'FullyQualifiedName !~ FailingConsumer' + + selftest-intended-failures: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # so that NerdBank.GitVersioning has access to history + - name: Install Nix + uses: cachix/install-nix-action@v30 + with: + extra_nix_config: | + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + - name: Restore dependencies + run: nix develop --command dotnet restore + - name: Build + run: 'nix develop --command dotnet build --no-restore --configuration Release' + - name: Test using self + run: 'nix develop --command dotnet exec ./WoofWare.NUnitTestRunner/bin/Release/net6.0/WoofWare.NUnitTestRunner.dll ./FailingConsumer/bin/Release/net8.0/FailingConsumer.dll --trx TrxOut/out.trx || true' + - name: Munge output + run: 'nix develop --command xmlstarlet sel -N x="http://microsoft.com/schemas/VisualStudio/TeamTest/2010" -t -m "//x:UnitTestResult" -v "@testName" -o ": " -v ".//x:ErrorInfo/x:Message" -n TrxOut/out.trx > snapshot.txt' + - name: Check output matches expected + run: 'actual=$(cat snapshot.txt | sort) expected=$(cat FailingConsumer/expected.txt | sort) [ "$expected" == "$actual" ]' selftest: runs-on: ubuntu-latest diff --git a/FailingConsumer/FailingConsumer.fsproj b/FailingConsumer/FailingConsumer.fsproj new file mode 100644 index 0000000..bd7d54b --- /dev/null +++ b/FailingConsumer/FailingConsumer.fsproj @@ -0,0 +1,20 @@ + + + + net8.0 + false + true + + + + + + + + + + + + + + diff --git a/FailingConsumer/TestInsufficientArgs.fs b/FailingConsumer/TestInsufficientArgs.fs new file mode 100644 index 0000000..406f178 --- /dev/null +++ b/FailingConsumer/TestInsufficientArgs.fs @@ -0,0 +1,9 @@ +namespace FailingConsumer + +open NUnit.Framework + +[] +module TestInsufficientArgs = + + [] + let foo (_ : int) = () diff --git a/FailingConsumer/expected.txt b/FailingConsumer/expected.txt new file mode 100644 index 0000000..a698959 --- /dev/null +++ b/FailingConsumer/expected.txt @@ -0,0 +1 @@ +foo: had parameter count mismatch: expected 1, actual 0 diff --git a/WoofWare.NUnitTestRunner.Lib/CreateTrxReport.fs b/WoofWare.NUnitTestRunner.Lib/CreateTrxReport.fs index 2e4113d..fecd8cf 100644 --- a/WoofWare.NUnitTestRunner.Lib/CreateTrxReport.fs +++ b/WoofWare.NUnitTestRunner.Lib/CreateTrxReport.fs @@ -164,6 +164,18 @@ module BuildTrxReport = | Some s -> s (Some stackTrace, message) + | TestFailure.TestFailed (UserMethodFailure.BadParameters (_, expected, actual)) + | TestFailure.SetUpFailed (UserMethodFailure.BadParameters (_, expected, actual)) + | TestFailure.TearDownFailed (UserMethodFailure.BadParameters (_, expected, actual)) -> + let newMessage = + $"had parameter count mismatch: expected %i{expected.Length}, actual %i{actual.Length}" + + let message = + match message with + | None -> newMessage + | Some message -> $"%s{message}\n%s{newMessage}" + + (stackTrace, Some message) | TestFailure.TestFailed (UserMethodFailure.ReturnedNonUnit (_, ret)) | TestFailure.SetUpFailed (UserMethodFailure.ReturnedNonUnit (_, ret)) | TestFailure.TearDownFailed (UserMethodFailure.ReturnedNonUnit (_, ret)) -> @@ -188,6 +200,14 @@ module BuildTrxReport = Message = None } |> Some + | Choice3Of3 (UserMethodFailure.BadParameters (_, expected, actual)) -> + { + StackTrace = None + Message = + $"parameter count mismatch, expected %i{expected.Length}, actual %i{actual.Length}" + |> Some + } + |> Some | Choice3Of3 (UserMethodFailure.ReturnedNonUnit (_, ret)) -> { Message = $"returned non-unit value %O{ret}" |> Some diff --git a/WoofWare.NUnitTestRunner.Lib/Domain.fs b/WoofWare.NUnitTestRunner.Lib/Domain.fs index 2a0f9ed..22d1b51 100644 --- a/WoofWare.NUnitTestRunner.Lib/Domain.fs +++ b/WoofWare.NUnitTestRunner.Lib/Domain.fs @@ -162,6 +162,8 @@ type UserMethodFailure = | ReturnedNonUnit of name : string * result : obj /// A method threw. | Threw of name : string * exn + /// Parameter count mismatch. + | BadParameters of name : string * expected : Type[] * actual : obj[] /// Human-readable representation of the user failure. override this.ToString () = @@ -170,12 +172,22 @@ type UserMethodFailure = $"User-defined method '%s{method}' returned a non-unit: %O{ret}" | UserMethodFailure.Threw (method, exc) -> $"User-defined method '%s{method}' threw: %s{exc.Message}\n %s{exc.StackTrace}" + | UserMethodFailure.BadParameters (method, expected, actual) -> + let expectedStr = expected |> Seq.map (fun t -> t.Name) |> String.concat ", " + + let actualStr = + actual + |> Seq.map (fun s -> if isNull s then "null" else s.ToString ()) + |> String.concat ", " + + $"User-defined method '%s{method}' had parameter count mismatch. Expected: (%s{expectedStr}) (%i{expected.Length} params). Actual: (%s{actualStr}) (%i{actual.Length} params)" /// Name (not fully-qualified) of the method which failed. member this.Name = match this with | UserMethodFailure.Threw (name, _) | UserMethodFailure.ReturnedNonUnit (name, _) -> name + | UserMethodFailure.BadParameters (name, _, _) -> name /// Represents the failure of a single run of one test. An error signalled this way is a user error: the unit under /// test has misbehaved. diff --git a/WoofWare.NUnitTestRunner.Lib/ParallelQueue.fs b/WoofWare.NUnitTestRunner.Lib/ParallelQueue.fs index 4095c31..de7901d 100644 --- a/WoofWare.NUnitTestRunner.Lib/ParallelQueue.fs +++ b/WoofWare.NUnitTestRunner.Lib/ParallelQueue.fs @@ -314,7 +314,7 @@ type ParallelQueue let t () = { new ThunkEvaluator<_> with member _.Eval<'b> (t : unit -> 'b) rc = - let tcs = TaskCompletionSource () + let tcs = TaskCompletionSource TaskCreationOptions.RunContinuationsAsynchronously use ec = ExecutionContext.Capture () fun () -> diff --git a/WoofWare.NUnitTestRunner.Lib/SurfaceBaseline.txt b/WoofWare.NUnitTestRunner.Lib/SurfaceBaseline.txt index 1ae2d3c..cad9026 100644 --- a/WoofWare.NUnitTestRunner.Lib/SurfaceBaseline.txt +++ b/WoofWare.NUnitTestRunner.Lib/SurfaceBaseline.txt @@ -703,13 +703,21 @@ WoofWare.NUnitTestRunner.TrxUnitTestResult.TestId [property]: [read-only] System WoofWare.NUnitTestRunner.TrxUnitTestResult.TestListId [property]: [read-only] System.Guid WoofWare.NUnitTestRunner.TrxUnitTestResult.TestName [property]: [read-only] string WoofWare.NUnitTestRunner.TrxUnitTestResult.TestType [property]: [read-only] System.Guid -WoofWare.NUnitTestRunner.UserMethodFailure inherit obj, implements WoofWare.NUnitTestRunner.UserMethodFailure System.IEquatable, System.Collections.IStructuralEquatable - union type with 2 cases +WoofWare.NUnitTestRunner.UserMethodFailure inherit obj, implements WoofWare.NUnitTestRunner.UserMethodFailure System.IEquatable, System.Collections.IStructuralEquatable - union type with 3 cases +WoofWare.NUnitTestRunner.UserMethodFailure+BadParameters inherit WoofWare.NUnitTestRunner.UserMethodFailure +WoofWare.NUnitTestRunner.UserMethodFailure+BadParameters.actual [property]: [read-only] obj [] +WoofWare.NUnitTestRunner.UserMethodFailure+BadParameters.expected [property]: [read-only] System.Type [] +WoofWare.NUnitTestRunner.UserMethodFailure+BadParameters.get_actual [method]: unit -> obj [] +WoofWare.NUnitTestRunner.UserMethodFailure+BadParameters.get_expected [method]: unit -> System.Type [] +WoofWare.NUnitTestRunner.UserMethodFailure+BadParameters.get_name [method]: unit -> string +WoofWare.NUnitTestRunner.UserMethodFailure+BadParameters.name [property]: [read-only] string WoofWare.NUnitTestRunner.UserMethodFailure+ReturnedNonUnit inherit WoofWare.NUnitTestRunner.UserMethodFailure WoofWare.NUnitTestRunner.UserMethodFailure+ReturnedNonUnit.get_name [method]: unit -> string WoofWare.NUnitTestRunner.UserMethodFailure+ReturnedNonUnit.get_result [method]: unit -> obj WoofWare.NUnitTestRunner.UserMethodFailure+ReturnedNonUnit.name [property]: [read-only] string WoofWare.NUnitTestRunner.UserMethodFailure+ReturnedNonUnit.result [property]: [read-only] obj WoofWare.NUnitTestRunner.UserMethodFailure+Tags inherit obj +WoofWare.NUnitTestRunner.UserMethodFailure+Tags.BadParameters [static field]: int = 2 WoofWare.NUnitTestRunner.UserMethodFailure+Tags.ReturnedNonUnit [static field]: int = 0 WoofWare.NUnitTestRunner.UserMethodFailure+Tags.Threw [static field]: int = 1 WoofWare.NUnitTestRunner.UserMethodFailure+Threw inherit WoofWare.NUnitTestRunner.UserMethodFailure @@ -718,13 +726,16 @@ WoofWare.NUnitTestRunner.UserMethodFailure+Threw.get_name [method]: unit -> stri WoofWare.NUnitTestRunner.UserMethodFailure+Threw.Item2 [property]: [read-only] System.Exception WoofWare.NUnitTestRunner.UserMethodFailure+Threw.name [property]: [read-only] string WoofWare.NUnitTestRunner.UserMethodFailure.Equals [method]: (WoofWare.NUnitTestRunner.UserMethodFailure, System.Collections.IEqualityComparer) -> bool +WoofWare.NUnitTestRunner.UserMethodFailure.get_IsBadParameters [method]: unit -> bool WoofWare.NUnitTestRunner.UserMethodFailure.get_IsReturnedNonUnit [method]: unit -> bool WoofWare.NUnitTestRunner.UserMethodFailure.get_IsThrew [method]: unit -> bool WoofWare.NUnitTestRunner.UserMethodFailure.get_Name [method]: unit -> string WoofWare.NUnitTestRunner.UserMethodFailure.get_Tag [method]: unit -> int +WoofWare.NUnitTestRunner.UserMethodFailure.IsBadParameters [property]: [read-only] bool WoofWare.NUnitTestRunner.UserMethodFailure.IsReturnedNonUnit [property]: [read-only] bool WoofWare.NUnitTestRunner.UserMethodFailure.IsThrew [property]: [read-only] bool WoofWare.NUnitTestRunner.UserMethodFailure.Name [property]: [read-only] string +WoofWare.NUnitTestRunner.UserMethodFailure.NewBadParameters [static method]: (string, System.Type [], obj []) -> WoofWare.NUnitTestRunner.UserMethodFailure WoofWare.NUnitTestRunner.UserMethodFailure.NewReturnedNonUnit [static method]: (string, obj) -> WoofWare.NUnitTestRunner.UserMethodFailure WoofWare.NUnitTestRunner.UserMethodFailure.NewThrew [static method]: (string, System.Exception) -> WoofWare.NUnitTestRunner.UserMethodFailure WoofWare.NUnitTestRunner.UserMethodFailure.Tag [property]: [read-only] int \ No newline at end of file diff --git a/WoofWare.NUnitTestRunner.Lib/TestFixture.fs b/WoofWare.NUnitTestRunner.Lib/TestFixture.fs index 7f991a6..9dc13dd 100644 --- a/WoofWare.NUnitTestRunner.Lib/TestFixture.fs +++ b/WoofWare.NUnitTestRunner.Lib/TestFixture.fs @@ -90,8 +90,15 @@ module TestFixture = let result = try head.Invoke (containingObject, args) |> Ok - with :? TargetInvocationException as e -> - Error (UserMethodFailure.Threw (head.Name, e.InnerException)) + with + | :? TargetInvocationException as e -> Error (UserMethodFailure.Threw (head.Name, e.InnerException)) + | :? TargetParameterCountException -> + UserMethodFailure.BadParameters ( + head.Name, + head.GetParameters () |> Array.map (fun pm -> pm.ParameterType), + args + ) + |> Error match result with | Error e -> Error (wrap e) diff --git a/WoofWare.NUnitTestRunner.Lib/version.json b/WoofWare.NUnitTestRunner.Lib/version.json index d3b2f88..0fb398d 100644 --- a/WoofWare.NUnitTestRunner.Lib/version.json +++ b/WoofWare.NUnitTestRunner.Lib/version.json @@ -1,5 +1,5 @@ { - "version": "0.18", + "version": "0.19", "publicReleaseRefSpec": [ "^refs/heads/main$" ], @@ -8,4 +8,4 @@ ":/Directory.Build.props", ":/README.md" ] -} \ No newline at end of file +} diff --git a/WoofWare.NUnitTestRunner.sln b/WoofWare.NUnitTestRunner.sln index c04fd13..cf1358c 100644 --- a/WoofWare.NUnitTestRunner.sln +++ b/WoofWare.NUnitTestRunner.sln @@ -12,6 +12,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WoofWare.NUnitTestRunner.St EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WoofWare.NUnitTestRunner.StartupHookLogic", "WoofWare.NUnitTestRunner.StartupHookLogic\WoofWare.NUnitTestRunner.StartupHookLogic.csproj", "{A70627C8-9D19-42C2-AFEB-CFBDDDCE045D}" EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FailingConsumer", "FailingConsumer\FailingConsumer.fsproj", "{DA7160F5-4C3C-4D2E-918B-7DCBA3F4272E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -42,5 +44,9 @@ Global {A70627C8-9D19-42C2-AFEB-CFBDDDCE045D}.Debug|Any CPU.Build.0 = Debug|Any CPU {A70627C8-9D19-42C2-AFEB-CFBDDDCE045D}.Release|Any CPU.ActiveCfg = Release|Any CPU {A70627C8-9D19-42C2-AFEB-CFBDDDCE045D}.Release|Any CPU.Build.0 = Release|Any CPU + {DA7160F5-4C3C-4D2E-918B-7DCBA3F4272E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DA7160F5-4C3C-4D2E-918B-7DCBA3F4272E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DA7160F5-4C3C-4D2E-918B-7DCBA3F4272E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DA7160F5-4C3C-4D2E-918B-7DCBA3F4272E}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/flake.nix b/flake.nix index ff411d6..0648ad0 100644 --- a/flake.nix +++ b/flake.nix @@ -63,6 +63,7 @@ pkgs.alejandra pkgs.nodePackages.markdown-link-check pkgs.shellcheck + pkgs.xmlstarlet ]; }; net6 = pkgs.mkShell {