patrick/WoofWare.NUnitTestRunner
1
0
Fork 0
mirror of https://github.com/Smaug123/unofficial-nunit-runner synced 2025-10-08 10:38:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement TRX files (#44)

Browse Source
This commit is contained in:
Patrick Stevens
2024-06-08 22:36:28 +01:00
committed by GitHub
parent 7c38982fe7
commit 21d340e450
6 changed files with 583 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
.github/workflows/dotnet.yaml vendored
View File

@@ -41,13 +41,21 @@ jobs:
run: 'nix develop --command dotnet test --no-build --verbosity normal --configuration ${{matrix.config}}'
selftest:
strategy:
matrix:
config:
- Release
- Debug
runs-on: ubuntu-latest
permissions:
actions: none
checks: write
contents: read
deployments: none
id-token: none
issues: none
discussions: none
packages: none
pages: none
pull-requests: read
repository-projects: none
security-events: none
statuses: read
steps:
- uses: actions/checkout@v4
@@ -61,9 +69,15 @@ jobs:
- name: Restore dependencies
run: nix develop --command dotnet restore
- name: Build
run: 'nix develop --command dotnet build --no-restore --configuration ${{matrix.config}}'
run: 'nix develop --command dotnet build --no-restore --configuration Release'
- name: Test using self
run: 'nix develop --command dotnet exec ./TestRunner/bin/${{matrix.config}}/net8.0/TestRunner.dll ./Consumer/bin/${{matrix.config}}/net8.0/Consumer.dll'
run: 'nix develop --command dotnet exec ./TestRunner/bin/Release/net8.0/TestRunner.dll ./Consumer/bin/Release/net8.0/Consumer.dll --trx TrxOut/out.trx'
- name: Parse Trx files
uses: NasAmin/trx-parser@v0.6.0
id: trx-parser
with:
TRX_PATH: ${{ github.workspace }}/TrxOut
REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
analyzers:
runs-on: ubuntu-latest

49
TestRunner.Lib/SurfaceBaseline.txt
View File

@@ -66,13 +66,37 @@ TestRunner.FilterModule inherit obj
TestRunner.FilterModule.parse [static method]: string -> TestRunner.Filter
TestRunner.FilterModule.shouldRun [static method]: TestRunner.Filter -> (TestRunner.TestFixture -> TestRunner.SingleTestMethod -> bool)
TestRunner.FixtureRunResults inherit obj, implements TestRunner.FixtureRunResults System.IEquatable, System.Collections.IStructuralEquatable
TestRunner.FixtureRunResults..ctor [constructor]: (TestRunner.TestMemberFailure list, int, TestRunner.UserMethodFailure list)
TestRunner.FixtureRunResults.Failed [property]: [read-only] TestRunner.TestMemberFailure list
TestRunner.FixtureRunResults.get_Failed [method]: unit -> TestRunner.TestMemberFailure list
TestRunner.FixtureRunResults.get_OtherFailures [method]: unit -> TestRunner.UserMethodFailure list
TestRunner.FixtureRunResults.get_SuccessCount [method]: unit -> int
TestRunner.FixtureRunResults.OtherFailures [property]: [read-only] TestRunner.UserMethodFailure list
TestRunner.FixtureRunResults.SuccessCount [property]: [read-only] int
TestRunner.FixtureRunResults..ctor [constructor]: ((TestRunner.TestMemberFailure * TestRunner.IndividualTestRunMetadata) list, (TestRunner.SingleTestMethod * TestRunner.TestMemberSuccess * TestRunner.IndividualTestRunMetadata) list, (TestRunner.UserMethodFailure * TestRunner.IndividualTestRunMetadata) list)
TestRunner.FixtureRunResults.Failed [property]: [read-only] (TestRunner.TestMemberFailure * TestRunner.IndividualTestRunMetadata) list
TestRunner.FixtureRunResults.get_Failed [method]: unit -> (TestRunner.TestMemberFailure * TestRunner.IndividualTestRunMetadata) list
TestRunner.FixtureRunResults.get_IndividualTestRunMetadata [method]: unit -> (TestRunner.IndividualTestRunMetadata * Microsoft.FSharp.Core.FSharpChoice<TestRunner.TestMemberFailure, TestRunner.TestMemberSuccess, TestRunner.UserMethodFailure>) list
TestRunner.FixtureRunResults.get_OtherFailures [method]: unit -> (TestRunner.UserMethodFailure * TestRunner.IndividualTestRunMetadata) list
TestRunner.FixtureRunResults.get_Success [method]: unit -> (TestRunner.SingleTestMethod * TestRunner.TestMemberSuccess * TestRunner.IndividualTestRunMetadata) list
TestRunner.FixtureRunResults.IndividualTestRunMetadata [property]: [read-only] (TestRunner.IndividualTestRunMetadata * Microsoft.FSharp.Core.FSharpChoice<TestRunner.TestMemberFailure, TestRunner.TestMemberSuccess, TestRunner.UserMethodFailure>) list
TestRunner.FixtureRunResults.OtherFailures [property]: [read-only] (TestRunner.UserMethodFailure * TestRunner.IndividualTestRunMetadata) list
TestRunner.FixtureRunResults.Success [property]: [read-only] (TestRunner.SingleTestMethod * TestRunner.TestMemberSuccess * TestRunner.IndividualTestRunMetadata) list
TestRunner.IndividualTestRunMetadata inherit obj, implements TestRunner.IndividualTestRunMetadata System.IEquatable, System.Collections.IStructuralEquatable, TestRunner.IndividualTestRunMetadata System.IComparable, System.IComparable, System.Collections.IStructuralComparable
TestRunner.IndividualTestRunMetadata..ctor [constructor]: (System.TimeSpan, System.DateTimeOffset, System.DateTimeOffset, string, System.Guid, System.Guid, string, string, string option, string option)
TestRunner.IndividualTestRunMetadata.ClassName [property]: [read-only] string
TestRunner.IndividualTestRunMetadata.ComputerName [property]: [read-only] string
TestRunner.IndividualTestRunMetadata.End [property]: [read-only] System.DateTimeOffset
TestRunner.IndividualTestRunMetadata.ExecutionId [property]: [read-only] System.Guid
TestRunner.IndividualTestRunMetadata.get_ClassName [method]: unit -> string
TestRunner.IndividualTestRunMetadata.get_ComputerName [method]: unit -> string
TestRunner.IndividualTestRunMetadata.get_End [method]: unit -> System.DateTimeOffset
TestRunner.IndividualTestRunMetadata.get_ExecutionId [method]: unit -> System.Guid
TestRunner.IndividualTestRunMetadata.get_Start [method]: unit -> System.DateTimeOffset
TestRunner.IndividualTestRunMetadata.get_StdErr [method]: unit -> string option
TestRunner.IndividualTestRunMetadata.get_StdOut [method]: unit -> string option
TestRunner.IndividualTestRunMetadata.get_TestId [method]: unit -> System.Guid
TestRunner.IndividualTestRunMetadata.get_TestName [method]: unit -> string
TestRunner.IndividualTestRunMetadata.get_Total [method]: unit -> System.TimeSpan
TestRunner.IndividualTestRunMetadata.Start [property]: [read-only] System.DateTimeOffset
TestRunner.IndividualTestRunMetadata.StdErr [property]: [read-only] string option
TestRunner.IndividualTestRunMetadata.StdOut [property]: [read-only] string option
TestRunner.IndividualTestRunMetadata.TestId [property]: [read-only] System.Guid
TestRunner.IndividualTestRunMetadata.TestName [property]: [read-only] string
TestRunner.IndividualTestRunMetadata.Total [property]: [read-only] System.TimeSpan
TestRunner.ITestProgress - interface with 5 member(s)
TestRunner.ITestProgress.OnTestFailed [method]: string -> TestRunner.TestMemberFailure -> unit
TestRunner.ITestProgress.OnTestFixtureStart [method]: string -> int -> unit
@@ -256,6 +280,10 @@ TestRunner.TestProgress.toStderr [static method]: unit -> TestRunner.ITestProgre
TestRunner.TrxCounters inherit obj, implements TestRunner.TrxCounters System.IEquatable, System.Collections.IStructuralEquatable
TestRunner.TrxCounters..ctor [constructor]: (System.UInt32, System.UInt32, System.UInt32, System.UInt32, System.UInt32, System.UInt32, System.UInt32, System.UInt32, System.UInt32, System.UInt32, System.UInt32, System.UInt32, System.UInt32, System.UInt32, System.UInt32, System.UInt32)
TestRunner.TrxCounters.Aborted [property]: [read-only] System.UInt32
TestRunner.TrxCounters.AddFailed [method]: unit -> TestRunner.TrxCounters
TestRunner.TrxCounters.AddInconclusive [method]: unit -> TestRunner.TrxCounters
TestRunner.TrxCounters.AddNotExecuted [method]: unit -> TestRunner.TrxCounters
TestRunner.TrxCounters.AddPassed [method]: unit -> TestRunner.TrxCounters
TestRunner.TrxCounters.Completed [property]: [read-only] System.UInt32
TestRunner.TrxCounters.Disconnected [property]: [read-only] System.UInt32
TestRunner.TrxCounters.Errors [property]: [read-only] System.UInt32
@@ -406,20 +434,25 @@ TestRunner.TrxTestMethod.get_ClassName [method]: unit -> string
TestRunner.TrxTestMethod.get_CodeBase [method]: unit -> string
TestRunner.TrxTestMethod.get_Name [method]: unit -> string
TestRunner.TrxTestMethod.Name [property]: [read-only] string
TestRunner.TrxTestOutcome inherit obj, implements TestRunner.TrxTestOutcome System.IEquatable, System.Collections.IStructuralEquatable - union type with 3 cases
TestRunner.TrxTestOutcome inherit obj, implements TestRunner.TrxTestOutcome System.IEquatable, System.Collections.IStructuralEquatable - union type with 4 cases
TestRunner.TrxTestOutcome+Tags inherit obj
TestRunner.TrxTestOutcome+Tags.Failed [static field]: int = 1
TestRunner.TrxTestOutcome+Tags.Inconclusive [static field]: int = 3
TestRunner.TrxTestOutcome+Tags.NotExecuted [static field]: int = 2
TestRunner.TrxTestOutcome+Tags.Passed [static field]: int = 0
TestRunner.TrxTestOutcome.Failed [static property]: [read-only] TestRunner.TrxTestOutcome
TestRunner.TrxTestOutcome.get_Failed [static method]: unit -> TestRunner.TrxTestOutcome
TestRunner.TrxTestOutcome.get_Inconclusive [static method]: unit -> TestRunner.TrxTestOutcome
TestRunner.TrxTestOutcome.get_IsFailed [method]: unit -> bool
TestRunner.TrxTestOutcome.get_IsInconclusive [method]: unit -> bool
TestRunner.TrxTestOutcome.get_IsNotExecuted [method]: unit -> bool
TestRunner.TrxTestOutcome.get_IsPassed [method]: unit -> bool
TestRunner.TrxTestOutcome.get_NotExecuted [static method]: unit -> TestRunner.TrxTestOutcome
TestRunner.TrxTestOutcome.get_Passed [static method]: unit -> TestRunner.TrxTestOutcome
TestRunner.TrxTestOutcome.get_Tag [method]: unit -> int
TestRunner.TrxTestOutcome.Inconclusive [static property]: [read-only] TestRunner.TrxTestOutcome
TestRunner.TrxTestOutcome.IsFailed [property]: [read-only] bool
TestRunner.TrxTestOutcome.IsInconclusive [property]: [read-only] bool
TestRunner.TrxTestOutcome.IsNotExecuted [property]: [read-only] bool
TestRunner.TrxTestOutcome.IsPassed [property]: [read-only] bool
TestRunner.TrxTestOutcome.NotExecuted [static property]: [read-only] TestRunner.TrxTestOutcome

313
TestRunner.Lib/TestFixture.fs
View File

@@ -1,22 +1,75 @@
namespace TestRunner
open System
open System.Diagnostics
open System.IO
open System.Reflection
open System.Threading
open Microsoft.FSharp.Core
type private StdoutSetter (newStdout : StreamWriter, newStderr : StreamWriter) =
let oldStdout = Console.Out
let oldStderr = Console.Error
do
Console.SetOut newStdout
Console.SetError newStderr
interface IDisposable with
member _.Dispose () =
Console.SetOut oldStdout
Console.SetError oldStderr
/// Information about the circumstances of a run of a single test.
type IndividualTestRunMetadata =
{
/// How long the test took.
Total : TimeSpan
/// When the test started.
Start : DateTimeOffset
/// When the test ended.
End : DateTimeOffset
/// The Environment.MachineName of the computer on which the run happened.
ComputerName : string
/// An identifier for this run of this test.
ExecutionId : Guid
/// An identifier for this test (possibly shared across repeats of this exact test with the same args).
TestId : Guid
/// Human-readable string representing this individual single test run, including any parameters.
TestName : string
/// Name of the class from which this test derived
ClassName : string
/// Anything that was printed to stdout while the test ran.
StdOut : string option
/// Anything that was printed to stderr while the test ran.
StdErr : string option
}
/// The results of running a single TestFixture.
type FixtureRunResults =
{
/// These tests failed.
Failed : TestMemberFailure list
/// This many tests succeeded (including multiple runs of a single test, if specified).
SuccessCount : int
/// TODO: domain is squiffy, the TestMemberFailure wants to be instead a TestFailure
Failed : (TestMemberFailure * IndividualTestRunMetadata) list
/// These tests succeeded.
/// A given test method may appear many times in this list, if it represented many tests.
Success : (SingleTestMethod * TestMemberSuccess * IndividualTestRunMetadata) list
/// These failures occurred outside the context of a test - e.g. in setup or tear-down logic.
OtherFailures : UserMethodFailure list
OtherFailures : (UserMethodFailure * IndividualTestRunMetadata) list
}
/// Another view on the data contained in this object, transposed.
member this.IndividualTestRunMetadata
: (IndividualTestRunMetadata * Choice<TestMemberFailure, TestMemberSuccess, UserMethodFailure>) list =
[
for a, d in this.Failed do
yield d, Choice1Of3 a
for _, a, d in this.Success do
yield d, Choice2Of3 a
for a, d in this.OtherFailures do
yield d, Choice3Of3 a
]
/// A test fixture (usually represented by the [<TestFixture>]` attribute), which may contain many tests,
/// each of which may run many times.
[<RequireQualifiedAccess>]
@@ -28,10 +81,11 @@ module TestFixture =
let private runOne
(setUp : MethodInfo list)
(tearDown : MethodInfo list)
(testId : Guid)
(test : MethodInfo)
(containingObject : obj)
(args : obj[])
: Result<TestMemberSuccess, TestFailure list>
: Result<TestMemberSuccess, TestFailure list> * IndividualTestRunMetadata
=
let rec runMethods
(wrap : UserMethodFailure -> TestFailure)
@@ -55,12 +109,56 @@ module TestFixture =
| :? unit -> runMethods wrap rest args
| ret -> UserMethodFailure.ReturnedNonUnit (head.Name, ret) |> wrap |> Error
match runMethods TestFailure.SetUpFailed setUp [||] with
| Error e -> Error [ e ]
let start = DateTimeOffset.Now
use stdOutStream = new MemoryStream ()
use stdErrStream = new MemoryStream ()
use stdOut = new StreamWriter (stdOutStream)
use stdErr = new StreamWriter (stdErrStream)
use _ = new StdoutSetter (stdOut, stdErr)
let sw = Stopwatch.StartNew ()
let metadata () =
let name =
if args.Length = 0 then
test.Name
else
let argsStr = args |> Seq.map string<obj> |> String.concat ","
$"%s{test.Name}(%s{argsStr})"
{
End = DateTimeOffset.Now
Start = start
Total = sw.Elapsed
ComputerName = Environment.MachineName
ExecutionId = Guid.NewGuid ()
TestId = testId
TestName = name
ClassName = test.DeclaringType.FullName
StdOut =
match stdOutStream.ToArray () with
| [||] -> None
| arr -> Console.OutputEncoding.GetString arr |> Some
StdErr =
match stdErrStream.ToArray () with
| [||] -> None
| arr -> Console.OutputEncoding.GetString arr |> Some
}
let setUpResult = runMethods TestFailure.SetUpFailed setUp [||]
sw.Stop ()
match setUpResult with
| Error e -> Error [ e ], metadata ()
| Ok () ->
sw.Start ()
let result =
let result = runMethods TestFailure.TestFailed [ test ] args
sw.Stop ()
match result with
| Ok () -> Ok None
@@ -76,14 +174,18 @@ module TestFixture =
| Error orig -> Error orig
// Unconditionally run TearDown after tests, even if tests failed.
sw.Start ()
let tearDownResult = runMethods TestFailure.TearDownFailed tearDown [||]
sw.Stop ()
let metadata = metadata ()
match result, tearDownResult with
| Ok None, Ok () -> Ok TestMemberSuccess.Ok
| Ok (Some s), Ok () -> Ok s
| Ok None, Ok () -> Ok TestMemberSuccess.Ok, metadata
| Ok (Some s), Ok () -> Ok s, metadata
| Error e, Ok ()
| Ok _, Error e -> Error [ e ]
| Error e1, Error e2 -> Error [ e1 ; e2 ]
| Ok _, Error e -> Error [ e ], metadata
| Error e1, Error e2 -> Error [ e1 ; e2 ], metadata
let private getValues (test : SingleTestMethod) =
let valuesAttrs =
@@ -126,7 +228,7 @@ module TestFixture =
(tearDown : MethodInfo list)
(containingObject : obj)
(test : SingleTestMethod)
: Result<TestMemberSuccess, TestMemberFailure> list
: (Result<TestMemberSuccess, TestMemberFailure> * IndividualTestRunMetadata) list
=
let resultPreRun =
(None, test.Modifiers)
@@ -141,45 +243,46 @@ module TestFixture =
| Modifier.Ignored reason -> Some (TestMemberSuccess.Ignored reason)
)
let sw = Stopwatch.StartNew ()
let startTime = DateTimeOffset.Now
match resultPreRun with
| Some result -> [ Ok result ]
| Some result ->
sw.Stop ()
let failureMetadata =
{
Total = sw.Elapsed
Start = startTime
End = DateTimeOffset.Now
ComputerName = Environment.MachineName
ExecutionId = Guid.NewGuid ()
// No need to keep these test GUIDs stable: no point trying to run an explicit test multiple times.
TestId = Guid.NewGuid ()
TestName = test.Name
ClassName = test.Method.DeclaringType.FullName
StdErr = None
StdOut = None
}
[ Ok result, failureMetadata ]
| None ->
Seq.init
(Option.defaultValue 1 test.Repeat)
(fun _ ->
let values = getValues test
match values with
| Error e -> Seq.singleton (Error e)
| Ok values ->
let inline normaliseError
(e : Result<TestMemberSuccess, TestFailure list>)
: Result<TestMemberSuccess, TestMemberFailure>
=
match e with
| Ok s -> Ok s
| Error e -> Error (e |> TestMemberFailure.Failed)
let individualTests =
let values = getValues test
match values with
| Error e -> Error e
| Ok values ->
match test.Kind, values with
| TestKind.Data data, None ->
data
|> Seq.map (fun args ->
runOne setUp tearDown test.Method containingObject (Array.ofList args)
|> normaliseError
)
| TestKind.Data data, None -> data |> List.map (fun args -> Guid.NewGuid (), Array.ofList args) |> Ok
| TestKind.Data _, Some _ ->
[
"Test has both the TestCase and Values attributes. Specify one or the other."
]
|> TestMemberFailure.Malformed
|> Error
|> Seq.singleton
| TestKind.Single, None ->
runOne setUp tearDown test.Method containingObject [||]
|> normaliseError
|> Seq.singleton
| TestKind.Single, None -> (Guid.NewGuid (), [||]) |> List.singleton |> Ok
| TestKind.Single, Some vals ->
let combinatorial =
Option.defaultValue Combinatorial.Combinatorial test.Combinatorial
@@ -190,30 +293,29 @@ module TestFixture =
|> Seq.map (fun l -> l |> Seq.map (fun v -> v.Value) |> Seq.toList)
|> Seq.toList
|> List.combinations
|> Seq.map (fun args ->
runOne setUp tearDown test.Method containingObject (Array.ofList args)
|> normaliseError
)
|> List.map (fun args -> Guid.NewGuid (), Array.ofList args)
|> Ok
| Combinatorial.Sequential ->
let maxLength = vals |> Seq.map (fun i -> i.Count) |> Seq.max
seq {
for i = 0 to maxLength - 1 do
List.init
maxLength
(fun i ->
let args =
vals
|> Array.map (fun param -> if i >= param.Count then null else param.[i].Value)
yield runOne setUp tearDown test.Method containingObject args |> normaliseError
}
Guid.NewGuid (), args
)
|> Ok
| TestKind.Source _, Some _ ->
[
"Test has both the TestCaseSource and Values attributes. Specify one or the other."
]
|> TestMemberFailure.Malformed
|> Error
|> Seq.singleton
| TestKind.Source sources, None ->
seq {
[
for source in sources do
let args =
test.Method.DeclaringType.GetProperty (
@@ -228,13 +330,11 @@ module TestFixture =
// Concretely, `FSharpList<HttpStatusCode> :> IEnumerable<obj>` fails.
for arg in args.GetValue (null : obj) :?> System.Collections.IEnumerable do
yield
Guid.NewGuid (),
match arg with
| :? Tuple<obj, obj> as (a, b) ->
runOne setUp tearDown test.Method containingObject [| a ; b |]
| :? Tuple<obj, obj, obj> as (a, b, c) ->
runOne setUp tearDown test.Method containingObject [| a ; b ; c |]
| :? Tuple<obj, obj, obj, obj> as (a, b, c, d) ->
runOne setUp tearDown test.Method containingObject [| a ; b ; c ; d |]
| :? Tuple<obj, obj> as (a, b) -> [| a ; b |]
| :? Tuple<obj, obj, obj> as (a, b, c) -> [| a ; b ; c |]
| :? Tuple<obj, obj, obj, obj> as (a, b, c, d) -> [| a ; b ; c ; d |]
| arg ->
let argTy = arg.GetType ()
@@ -250,18 +350,47 @@ module TestFixture =
if isNull argsMem then
failwith "Unexpectedly could not call `.Arguments` on TestCaseData"
runOne
setUp
tearDown
test.Method
containingObject
(argsMem.Invoke (arg, [||]) |> unbox<obj[]>)
(argsMem.Invoke (arg, [||]) |> unbox<obj[]>)
else
runOne setUp tearDown test.Method containingObject [| arg |]
|> normaliseError
}
)
[| arg |]
]
|> Ok
sw.Stop ()
match individualTests with
| Error e ->
let failureMetadata =
{
Total = sw.Elapsed
Start = startTime
End = DateTimeOffset.Now
ComputerName = Environment.MachineName
ExecutionId = Guid.NewGuid ()
// No need to keep these test GUIDs stable: we're not going to run them multiple times,
// because we're not going to run anything at all.
TestId = Guid.NewGuid ()
TestName = test.Name
ClassName = test.Method.DeclaringType.FullName
StdErr = None
StdOut = None
}
[ Error e, failureMetadata ]
| Ok individualTests ->
let count = test.Repeat |> Option.defaultValue 1
Seq.init count (fun _ -> individualTests)
|> Seq.concat
|> Seq.map (fun (testGuid, args) ->
let results, summary =
runOne setUp tearDown testGuid test.Method containingObject args
match results with
| Ok results -> Ok results, summary
| Error e -> Error (TestMemberFailure.Failed e), summary
)
|> Seq.toList
/// Run every test (except those which fail the `filter`) in this test fixture, as well as the
@@ -300,19 +429,48 @@ module TestFixture =
let oldWorkDir = Environment.CurrentDirectory
Environment.CurrentDirectory <- FileInfo(tests.ContainingAssembly.Location).Directory.FullName
let sw = Stopwatch.StartNew ()
let startTime = DateTimeOffset.UtcNow
use stdOutStream = new MemoryStream ()
use stdOut = new StreamWriter (stdOutStream)
use stdErrStream = new MemoryStream ()
use stdErr = new StreamWriter (stdErrStream)
use _ = new StdoutSetter (stdOut, stdErr)
let endMetadata () =
let stdOut = stdOutStream.ToArray () |> Console.OutputEncoding.GetString
let stdErr = stdErrStream.ToArray () |> Console.OutputEncoding.GetString
{
Total = sw.Elapsed
Start = startTime
End = DateTimeOffset.UtcNow
ComputerName = Environment.MachineName
ExecutionId = Guid.NewGuid ()
TestId = Guid.NewGuid ()
// This one is a bit dubious, because we don't actually have a test name at all
TestName = tests.Name
ClassName = tests.Name
StdOut = if String.IsNullOrEmpty stdOut then None else Some stdOut
StdErr = if String.IsNullOrEmpty stdErr then None else Some stdErr
}
let setupResult =
match tests.OneTimeSetUp with
| Some su ->
try
match su.Invoke (containingObject, [||]) with
| :? unit -> None
| ret -> Some (UserMethodFailure.ReturnedNonUnit (su.Name, ret))
| ret -> Some (UserMethodFailure.ReturnedNonUnit (su.Name, ret), endMetadata ())
with :? TargetInvocationException as e ->
Some (UserMethodFailure.Threw (su.Name, e.InnerException))
Some (UserMethodFailure.Threw (su.Name, e.InnerException), endMetadata ())
| _ -> None
let totalTestSuccess = ref 0
let testFailures = ResizeArray ()
let testFailures = ResizeArray<TestMemberFailure * IndividualTestRunMetadata> ()
let successes =
ResizeArray<SingleTestMethod * TestMemberSuccess * IndividualTestRunMetadata> ()
match setupResult with
| Some _ ->
@@ -326,14 +484,15 @@ module TestFixture =
let results = runTestsFromMember tests.SetUp tests.TearDown containingObject test
for result in results do
for result, report in results do
match result with
| Error failure ->
testFailures.Add failure
testFailures.Add (failure, report)
progress.OnTestFailed test.Name failure
| Ok _ -> Interlocked.Increment testSuccess |> ignore<int>
| Ok result ->
Interlocked.Increment testSuccess |> ignore<int>
lock successes (fun () -> successes.Add (test, result, report))
Interlocked.Add (totalTestSuccess, testSuccess.Value) |> ignore<int>
progress.OnTestMemberFinished test.Name
else
progress.OnTestMemberSkipped test.Name
@@ -345,16 +504,16 @@ module TestFixture =
try
match td.Invoke (containingObject, [||]) with
| null -> None
| ret -> Some (UserMethodFailure.ReturnedNonUnit (td.Name, ret))
| ret -> Some (UserMethodFailure.ReturnedNonUnit (td.Name, ret), endMetadata ())
with :? TargetInvocationException as e ->
Some (UserMethodFailure.Threw (td.Name, e))
Some (UserMethodFailure.Threw (td.Name, e), endMetadata ())
| _ -> None
Environment.CurrentDirectory <- oldWorkDir
{
Failed = testFailures |> Seq.toList
SuccessCount = totalTestSuccess.Value
Success = successes |> Seq.toList
OtherFailures = [ tearDownError ; setupResult ] |> List.choose id
}

36
TestRunner.Lib/TrxReport.fs
View File

@@ -207,6 +207,8 @@ type TrxTestOutcome =
| Failed
/// The test was not executed.
| NotExecuted
/// The test was inconclusive. (This appears not to be modelled correctly by NUnit! They use NotExecuted.)
| Inconclusive
/// Serialisation suitable for direct interpolation into a TRX report.
override this.ToString () =
@@ -214,6 +216,7 @@ type TrxTestOutcome =
| TrxTestOutcome.Passed -> "Passed"
| TrxTestOutcome.Failed -> "Failed"
| TrxTestOutcome.NotExecuted -> "NotExecuted"
| TrxTestOutcome.Inconclusive -> "Inconclusive"
/// Round-trips with `ToString`; returns None if parse was unsuccessful.
static member Parse (s : string) : TrxTestOutcome option =
@@ -223,6 +226,8 @@ type TrxTestOutcome =
Some TrxTestOutcome.Failed
elif s.Equals ("notexecuted", StringComparison.OrdinalIgnoreCase) then
Some TrxTestOutcome.NotExecuted
elif s.Equals ("inconclusive", StringComparison.OrdinalIgnoreCase) then
Some TrxTestOutcome.Inconclusive
else
None
@@ -1117,6 +1122,37 @@ type TrxCounters =
Pending : uint
}
/// Create a new Counters with one more Passed test.
member this.AddPassed () =
{ this with
Passed = this.Passed + 1u
Total = this.Total + 1u
Executed = this.Executed + 1u
}
/// Create a new Counters with one more Inconclusive test.
member this.AddInconclusive () =
{ this with
Inconclusive = this.Inconclusive + 1u
Total = this.Total + 1u
Executed = this.Executed + 1u
}
/// Create a new Counters with one more NotExecuted test.
member this.AddNotExecuted () =
{ this with
NotExecuted = this.NotExecuted + 1u
Total = this.Total + 1u
}
/// Create a new Counters with one more Failed test.
member this.AddFailed () =
{ this with
Executed = this.Executed + 1u
Total = this.Total + 1u
Failed = this.Failed + 1u
}
member internal this.toXml (doc : XmlDocument) : XmlNode =
let node = doc.CreateElement "Counters"

2
TestRunner.Lib/version.json
View File

@@ -1,5 +1,5 @@
{
"version": "0.6",
"version": "0.7",
"publicReleaseRefSpec": null,
"pathFilters": [
"./",

274
TestRunner/Program.fs
View File

@@ -103,11 +103,20 @@ module Program =
| Some (Choice2Of2 sdk) -> [ dll.Directory ; DirectoryInfo sdk.Path ]
let main argv =
let testDll, filter =
let startTime = DateTimeOffset.Now
let testDll, filter, trxPath =
match argv |> List.ofSeq with
| [ dll ] -> FileInfo dll, None
| [ dll ; "--filter" ; filter ] -> FileInfo dll, Some (Filter.parse filter)
| _ -> failwith "provide exactly one arg, a test DLL"
| [ dll ] -> FileInfo dll, None, None
| [ dll ; "--trx" ; trxPath ] -> FileInfo dll, None, Some (FileInfo trxPath)
| [ dll ; "--filter" ; filter ] -> FileInfo dll, Some (Filter.parse filter), None
| [ dll ; "--trx" ; trxPath ; "--filter" ; filter ] ->
FileInfo dll, Some (Filter.parse filter), Some (FileInfo trxPath)
| [ dll ; "--filter" ; filter ; "--trx" ; trxPath ] ->
FileInfo dll, Some (Filter.parse filter), Some (FileInfo trxPath)
| _ ->
failwith
"provide exactly one arg, a test DLL; then optionally `--filter <filter>` and/or `--trx <output-filename>`."
let filter =
match filter with
@@ -121,37 +130,248 @@ module Program =
let ctx = Ctx (testDll, locateRuntimes testDll)
let assy = ctx.LoadFromAssemblyPath testDll.FullName
let anyFailures =
assy.ExportedTypes
|> Seq.fold
(fun anyFailures ty ->
let testFixture = TestFixture.parse ty
let testFixtures = assy.ExportedTypes |> Seq.map TestFixture.parse |> Seq.toList
let results = TestFixture.run progress filter testFixture
let creationTime = DateTimeOffset.Now
let results = testFixtures |> List.map (TestFixture.run progress filter)
let anyFailures =
match results.Failed with
| [] -> anyFailures
| _ :: _ ->
eprintfn $"%i{results.Failed.Length} tests failed"
true
let finishTime = DateTimeOffset.Now
let finishTimeHumanReadable = finishTime.ToString @"yyyy-MM-dd HH:mm:ss"
let nowMachine = finishTime.ToString @"yyyy-MM-dd_HH_mm_ss"
let anyFailures =
match results.OtherFailures with
| [] -> anyFailures
| otherFailures ->
eprintfn "Other failures encountered: "
let testListId = Guid.NewGuid ()
for failure in otherFailures do
eprintfn $" %s{failure.Name}"
let testDefinitions, testEntries =
results
|> List.collect (fun results -> results.IndividualTestRunMetadata)
|> List.map (fun (data, _) ->
let defn =
{
Name = data.TestName
Storage = assy.Location.ToLowerInvariant ()
Id = data.TestId
Execution =
{
Id = data.ExecutionId
}
TestMethod =
{
CodeBase = assy.Location
AdapterTypeName = Uri "executor://woofware/"
ClassName = data.ClassName
Name = data.TestName
}
}
true
let entry : TrxTestEntry =
{
TestListId = testListId
ExecutionId = data.ExecutionId
TestId = data.TestId
anyFailures
}
defn, entry
)
|> List.unzip
let hostname = Environment.MachineName
let settings =
{
Name = "default"
Id = Guid.NewGuid ()
Deployment =
{
RunDeploymentRoot = $"_%s{hostname}_%s{nowMachine}"
}
}
let testList : TrxTestListEntry =
{
Id = testListId
Name = "All"
}
let counters =
(TrxCounters.Zero, results)
// TODO: this is woefully inefficient
||> List.fold (fun counters results ->
let counters =
(counters, results.Failed)
||> List.fold (fun counters (_, _) ->
// TODO: the counters can be more specific about the failure mode
counters.AddFailed ()
)
let counters =
(counters, results.OtherFailures)
||> List.fold (fun counters _ ->
// TODO: the counters can be more specific about the failure mode
counters.AddFailed ()
)
(counters, results.Success)
||> List.fold (fun counters (_, success, _) ->
match success with
| TestMemberSuccess.Ok -> counters.AddPassed ()
| TestMemberSuccess.Ignored _
| TestMemberSuccess.Explicit _ -> counters.AddNotExecuted ()
| TestMemberSuccess.Inconclusive _ -> counters.AddInconclusive ()
)
false
)
if anyFailures then 1 else 0
// TODO: I'm sure we can do better than this; there's a whole range of possible
// states!
let outcome =
if counters.Failed > 0u then
TrxOutcome.Failed
else
TrxOutcome.Completed
let resultSummary : TrxResultsSummary =
{
Outcome = outcome
Counters = counters
Output =
{
StdOut = None
ErrorInfo = None
}
RunInfos =
[
// TODO: capture stdout
]
}
let times : TrxReportTimes =
{
Creation = creationTime
Queuing = startTime
Start = startTime
Finish = finishTime
}
let magicGuid = Guid.Parse "13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b"
let results =
results
|> List.collect (fun results -> results.IndividualTestRunMetadata)
|> List.map (fun (i, cause) ->
let exc =
match cause with
| Choice2Of3 _ -> None
| Choice1Of3 (TestMemberFailure.Malformed reasons) ->
{
StackTrace = None
Message = reasons |> String.concat "\n" |> Some
}
|> Some
| Choice1Of3 (TestMemberFailure.Failed fail)
| Choice1Of3 (TestMemberFailure.Failed fail)
| Choice1Of3 (TestMemberFailure.Failed fail) ->
((None, None), fail)
||> List.fold (fun (stackTrace, message) tf ->
match tf with
| TestFailure.TestFailed (UserMethodFailure.Threw (_, exc))
| TestFailure.SetUpFailed (UserMethodFailure.Threw (_, exc))
| TestFailure.TearDownFailed (UserMethodFailure.Threw (_, exc)) ->
let stackTrace =
match stackTrace with
| None -> (exc : Exception).ToString ()
| Some s -> s
(Some stackTrace, message)
| TestFailure.TestFailed (UserMethodFailure.ReturnedNonUnit (_, ret))
| TestFailure.SetUpFailed (UserMethodFailure.ReturnedNonUnit (_, ret))
| TestFailure.TearDownFailed (UserMethodFailure.ReturnedNonUnit (_, ret)) ->
let newMessage = $"returned non-unit value %O{ret}"
let message =
match message with
| None -> newMessage
| Some message -> $"%s{message}\n%s{newMessage}"
(stackTrace, Some message)
)
|> fun (stackTrace, message) ->
{
StackTrace = stackTrace
Message = message
}
|> Some
| Choice3Of3 (UserMethodFailure.Threw (_, exc)) ->
{
StackTrace = (exc : Exception).ToString () |> Some
Message = None
}
|> Some
| Choice3Of3 (UserMethodFailure.ReturnedNonUnit (_, ret)) ->
{
Message = $"returned non-unit value %O{ret}" |> Some
StackTrace = None
}
|> Some
let outcome =
match cause with
| Choice1Of3 _ -> TrxTestOutcome.Failed
| Choice2Of3 TestMemberSuccess.Ok -> TrxTestOutcome.Passed
| Choice2Of3 (TestMemberSuccess.Inconclusive _) -> TrxTestOutcome.Inconclusive
| Choice2Of3 (TestMemberSuccess.Ignored _)
| Choice2Of3 (TestMemberSuccess.Explicit _) -> TrxTestOutcome.NotExecuted
// TODO: we can totally do better here, more fine-grained classification
| Choice3Of3 _ -> TrxTestOutcome.Failed
{
ExecutionId = i.ExecutionId
TestId = i.TestId
TestName = i.TestName
ComputerName = i.ComputerName
Duration = i.End - i.Start
StartTime = i.Start
EndTime = i.End
TestType = magicGuid
Outcome = outcome
TestListId = testListId
RelativeResultsDirectory = i.ExecutionId.ToString () // for some reason
Output =
match i.StdOut, i.StdErr, exc with
| None, None, None -> None
// TODO surely stderr can be emitted
| stdout, _stderr, exc ->
Some
{
TrxOutput.StdOut = stdout
ErrorInfo = exc
}
}
)
let report : TrxReport =
{
Id = Guid.NewGuid ()
Name = $"@%s{hostname} %s{finishTimeHumanReadable}"
Times = times
Settings = settings
Results = results
TestDefinitions = testDefinitions
TestEntries = testEntries
TestLists = [ testList ]
ResultsSummary = resultSummary
}
match trxPath with
| Some trxPath ->
let contents = TrxReport.toXml report |> fun d -> d.OuterXml
trxPath.Directory.Create ()
File.WriteAllText (trxPath.FullName, contents)
| None -> ()
match outcome with
| TrxOutcome.Completed -> 0
| _ -> 1
[<EntryPoint>]
let reallyMain argv =