Compare commits

..

4 Commits

Author SHA1 Message Date
Patrick Stevens
8e7c54cc83 Split out AppContext and Args to lib (#93) 2024-06-23 15:26:35 +01:00
Patrick Stevens
378a0169f8 Formatting (#92) 2024-06-23 15:13:40 +01:00
Patrick Stevens
870804d6ef Multi-framework Consumer test (#89) 2024-06-23 11:50:58 +01:00
Patrick Stevens
9f5f22c644 Pull report generation into lib (#90) 2024-06-23 11:44:53 +01:00
13 changed files with 532 additions and 407 deletions

View File

@@ -1,40 +1,40 @@
root=true
root = true
[*]
charset=utf-8
trim_trailing_whitespace=true
insert_final_newline=true
indent_style=space
indent_size=4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
indent_style = space
indent_size = 4
# ReSharper properties
resharper_xml_indent_size=2
resharper_xml_max_line_length=100
resharper_xml_tab_width=2
resharper_xml_indent_size = 2
resharper_xml_max_line_length = 100
resharper_xml_tab_width = 2
[*.{csproj,fsproj,sqlproj,targets,props,ts,tsx,css,json}]
indent_style=space
indent_size=2
indent_style = space
indent_size = 2
[*.{fs,fsi}]
fsharp_bar_before_discriminated_union_declaration=true
fsharp_space_before_uppercase_invocation=true
fsharp_space_before_class_constructor=true
fsharp_space_before_member=true
fsharp_space_before_colon=true
fsharp_space_before_semicolon=true
fsharp_multiline_bracket_style=aligned
fsharp_newline_between_type_definition_and_members=true
fsharp_align_function_signature_to_indentation=true
fsharp_alternative_long_member_definitions=true
fsharp_multi_line_lambda_closing_newline=true
fsharp_experimental_keep_indent_in_branch=true
fsharp_max_value_binding_width=80
fsharp_max_record_width=0
max_line_length=120
end_of_line=lf
fsharp_bar_before_discriminated_union_declaration = true
fsharp_space_before_uppercase_invocation = true
fsharp_space_before_class_constructor = true
fsharp_space_before_member = true
fsharp_space_before_colon = true
fsharp_space_before_semicolon = true
fsharp_multiline_bracket_style = aligned
fsharp_newline_between_type_definition_and_members = true
fsharp_align_function_signature_to_indentation = true
fsharp_alternative_long_member_definitions = true
fsharp_multi_line_lambda_closing_newline = true
fsharp_experimental_keep_indent_in_branch = true
fsharp_max_value_binding_width = 80
fsharp_max_record_width = 0
max_line_length = 120
end_of_line = lf
[*.{appxmanifest,build,dtd,nuspec,xaml,xamlx,xoml,xsd}]
indent_style=space
indent_size=2
tab_width=2
indent_style = space
indent_size = 2
tab_width = 2

View File

@@ -57,6 +57,10 @@ jobs:
security-events: none
statuses: read
strategy:
matrix:
framework: ["net6.0", "net8.0"]
steps:
- uses: actions/checkout@v4
with:
@@ -71,7 +75,7 @@ jobs:
- name: Build
run: 'nix develop --command dotnet build --no-restore --configuration Release'
- name: Test using self
run: 'nix develop --command dotnet exec ./WoofWare.NUnitTestRunner/bin/Release/net8.0/WoofWare.NUnitTestRunner.dll ./Consumer/bin/Release/net8.0/Consumer.dll --trx TrxOut/out.trx'
run: 'nix develop --command dotnet exec ./WoofWare.NUnitTestRunner/bin/Release/net8.0/WoofWare.NUnitTestRunner.dll ./Consumer/bin/Release/${{matrix.framework}}/Consumer.dll --trx TrxOut/out.trx'
- name: Parse Trx files
uses: NasAmin/trx-parser@v0.6.0
if: always()

View File

@@ -1,8 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFrameworks>net6.0;net8.0</TargetFrameworks>
<IsPackable>false</IsPackable>
<IsTestProject>true</IsTestProject>
</PropertyGroup>
@@ -26,9 +25,9 @@
<ItemGroup>
<PackageReference Include="FsUnit" Version="6.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.10.0"/>
<PackageReference Include="NUnit" Version="4.1.0"/>
<PackageReference Include="NUnit3TestAdapter" Version="4.5.0"/>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.10.0"/>
<PackageReference Include="NUnit" Version="4.1.0"/>
<PackageReference Include="NUnit3TestAdapter" Version="4.5.0"/>
</ItemGroup>
</Project>

View File

@@ -0,0 +1,34 @@
namespace WoofWare.NUnitTestRunner
open System
open System.IO
open System.Reflection
// Fix for https://github.com/Smaug123/unofficial-nunit-runner/issues/8
// (This tells the DLL loader to look next to the test DLL for dependencies.)
// TODO: do we actually need this when we move to the Hooks world?
/// Context manager to set the AppContext.BaseDirectory of the executing DLL.
type SetBaseDir (testDll : FileInfo) =
let oldBaseDir = AppContext.BaseDirectory
let setData =
let appContext = Type.GetType "System.AppContext"
if Object.ReferenceEquals (appContext, (null : obj)) then
ignore<string * string>
else
let setDataMethod =
appContext.GetMethod ("SetData", BindingFlags.Static ||| BindingFlags.Public)
if Object.ReferenceEquals (setDataMethod, (null : obj)) then
ignore<string * string>
else
fun (k, v) -> setDataMethod.Invoke ((null : obj), [| k ; v |]) |> unbox<unit>
do setData ("APP_CONTEXT_BASE_DIRECTORY", testDll.Directory.FullName)
interface IDisposable with
member _.Dispose () =
setData ("APP_CONTEXT_BASE_DIRECTORY", oldBaseDir)

View File

@@ -0,0 +1,99 @@
namespace WoofWare.NUnitTestRunner
open System
open System.IO
[<AutoOpen>]
module internal Patterns =
let (|Key|_|) (start : string) (s : string) : string option =
if s.StartsWith (start + "=", StringComparison.Ordinal) then
s.Substring (start.Length + 1) |> Some
else
None
/// Represents how verbose the test runner's logging should be.
[<RequireQualifiedAccess>]
type LogLevel =
/// Don't log any information about the test run.
| Nothing
/// Log as much information as is available about the test run.
| Verbose
/// Arguments controlling the test runner itself (not the tests therein).
type Args =
{
/// The DLL containing the tests we'll reflectively discover and invoke.
Dll : FileInfo
/// If set, the output file into which we will write a TRX report. (We'll create parent directories as necessary.)
Trx : FileInfo option
/// Also contains the original string which specified the filter.
Filter : (string * Filter) option
/// How verbose to be with the test runner's own logging.
Logging : LogLevel
/// Maximum number of tests which can run concurrently. This setting overrides any LevelOfParallelism reflectively
/// extracted from the assembly under test.
LevelOfParallelism : int option
/// Abort if the test runner is running for longer than this timeout.
Timeout : TimeSpan option
}
/// Parse `argv` into a structured Args.
static member Parse (args : string list) : Args =
match args with
| [] -> failwith "The first arg must be a positional arg, the DLL to test."
| dll :: args ->
let rec go
(trx : FileInfo option)
(filter : (string * Filter) option)
(logging : LogLevel option)
(par : int option)
(timeout : TimeSpan option)
(args : string list)
=
match args with
| [] ->
{
Dll = FileInfo dll
Trx = trx
Filter = filter
Logging = logging |> Option.defaultValue LogLevel.Nothing
LevelOfParallelism = par
Timeout = timeout
}
| Key "--filter" filterStr :: rest
| "--filter" :: filterStr :: rest ->
match filter with
| Some _ -> failwith "Two conflicting filters; you can only specify --filter once"
| None -> go trx (Some (filterStr, Filter.parse filterStr)) logging par timeout rest
| Key "--trx" trxStr :: rest
| "--trx" :: trxStr :: rest ->
match trx with
| Some _ -> failwith "Two conflicting TRX outputs; you can only specify --trx once"
| None -> go (Some (FileInfo trxStr)) filter logging par timeout rest
| Key "--verbose" verboseStr :: rest
| "--verbose" :: verboseStr :: rest ->
match logging with
| Some _ -> failwith "Two conflicting --verbose outputs; you can only specify --verbose once"
| None ->
let verbose =
if Boolean.Parse verboseStr then
LogLevel.Verbose
else
LogLevel.Nothing
go trx filter (Some verbose) par timeout rest
| Key "--parallelism" parStr :: rest
| "--parallelism" :: parStr :: rest ->
match par with
| Some _ -> failwith "Two conflicting --parallelism outputs; you can only specify --parallelism once"
| None -> go trx filter logging (Some (Int32.Parse parStr)) timeout rest
| Key "--timeout-seconds" timeoutStr :: rest
| "--timeout-seconds" :: timeoutStr :: rest ->
match timeout with
| Some _ ->
failwith "Two conflicting --timeout-seconds outputs; you can only specify --timeout-seconds once"
| None -> go trx filter logging par (Some (TimeSpan.FromSeconds (Int32.Parse timeoutStr |> float))) rest
| k :: _rest -> failwith $"Unrecognised arg %s{k}"
go None None None None None args

View File

@@ -0,0 +1,58 @@
namespace WoofWare.NUnitTestRunner
open System.Reflection
/// Attributes at the assembly level which control the behaviour of NUnit.
type AssemblyLevelAttributes =
{
/// How many tests can be running at once, if anything's running in parallel.
Parallelism : int option
/// Whether the tests in this assembly can be parallelised at all.
Parallelizable : Parallelizable<AssemblyParallelScope> option
}
[<RequireQualifiedAccess>]
module AssemblyLevelAttributes =
/// Reflectively obtain the values of any relevant assembly attributes.
let get (assy : Assembly) : AssemblyLevelAttributes =
((None, None), assy.CustomAttributes)
||> Seq.fold (fun (levelPar, par) attr ->
match attr.AttributeType.FullName with
| "NUnit.Framework.LevelOfParallelismAttribute" ->
let arg = attr.ConstructorArguments |> Seq.exactlyOne |> _.Value |> unbox<int>
match levelPar with
| None -> (Some arg, par)
| Some existing ->
failwith $"Assembly %s{assy.Location} declares parallelism %i{arg} and also %i{existing}"
| "NUnit.Framework.NonParallelizableAttribute" ->
match levelPar with
| None -> (Some 1, par)
| Some existing ->
failwith
$"Assembly %s{assy.Location} declares non-parallelizable and also parallelism %i{existing}"
| "NUnit.Framework.ParallelizableAttribute" ->
match par with
| Some _ -> failwith "Got multiple Parallelize attributes in assembly"
| None ->
match attr.ConstructorArguments |> Seq.toList with
| [] -> levelPar, Some (Parallelizable.Yes AssemblyParallelScope.Fixtures)
| [ v ] ->
match v.Value with
| :? int as v ->
match v with
| 512 -> levelPar, Some (Parallelizable.Yes AssemblyParallelScope.Fixtures)
| 256 -> levelPar, Some (Parallelizable.Yes AssemblyParallelScope.Children)
| 257 -> failwith "ParallelScope.All is invalid on assemblies; only Fixtures or Children"
| 1 -> failwith "ParallelScope.Self is invalid on assemblies; only Fixtures or Children"
| v -> failwith $"Could not recognise value %i{v} of parallel scope on assembly"
| v -> failwith $"Unexpectedly non-int value %O{v} of parallel scope on assembly"
| _ -> failwith "unexpectedly got multiple args to Parallelizable on assembly"
| _ -> levelPar, par
)
|> fun (par, canPar) ->
{
Parallelizable = canPar
Parallelism = par
}

View File

@@ -0,0 +1,243 @@
namespace WoofWare.NUnitTestRunner
open System
open System.Reflection
/// Methods for constructing TRX reports.
[<RequireQualifiedAccess>]
module BuildTrxReport =
/// Build a TRX report from the given results.
let build
(assy : Assembly)
(creationTime : DateTimeOffset)
(startTime : DateTimeOffset)
(results : FixtureRunResults list)
: TrxReport
=
let finishTime = DateTimeOffset.Now
let finishTimeHumanReadable = finishTime.ToString @"yyyy-MM-dd HH:mm:ss"
let nowMachine = finishTime.ToString @"yyyy-MM-dd_HH_mm_ss"
let testListId = Guid.NewGuid ()
let testDefinitions, testEntries =
results
|> List.collect (fun results -> results.IndividualTestRunMetadata)
|> List.map (fun (data, _) ->
let defn =
{
Name = data.TestName
Storage = assy.Location.ToLowerInvariant ()
Id = data.TestId
Execution =
{
Id = data.ExecutionId
}
TestMethod =
{
CodeBase = assy.Location
AdapterTypeName = Uri "executor://woofware/"
ClassName = data.ClassName
Name = data.TestName
}
}
let entry : TrxTestEntry =
{
TestListId = testListId
ExecutionId = data.ExecutionId
TestId = data.TestId
}
defn, entry
)
|> List.unzip
let hostname = Environment.MachineName
let settings =
{
Name = "default"
Id = Guid.NewGuid ()
Deployment =
{
RunDeploymentRoot = $"_%s{hostname}_%s{nowMachine}"
}
}
let testList : TrxTestListEntry =
{
Id = testListId
Name = "All"
}
let counters =
(TrxCounters.Zero, results)
// TODO: this is woefully inefficient
||> List.fold (fun counters results ->
let counters =
(counters, results.Failed)
||> List.fold (fun counters (_, _) ->
// TODO: the counters can be more specific about the failure mode
counters.AddFailed ()
)
let counters =
(counters, results.OtherFailures)
||> List.fold (fun counters _ ->
// TODO: the counters can be more specific about the failure mode
counters.AddFailed ()
)
(counters, results.Success)
||> List.fold (fun counters (_, success, _) ->
match success with
| TestMemberSuccess.Ok -> counters.AddPassed ()
| TestMemberSuccess.Ignored _
| TestMemberSuccess.Explicit _ -> counters.AddNotExecuted ()
| TestMemberSuccess.Inconclusive _ -> counters.AddInconclusive ()
)
)
// TODO: I'm sure we can do better than this; there's a whole range of possible
// states!
let outcome =
if counters.Failed > 0u then
TrxOutcome.Failed
else
TrxOutcome.Completed
let resultSummary : TrxResultsSummary =
{
Outcome = outcome
Counters = counters
Output =
{
StdOut = None
StdErr = None
ErrorInfo = None
}
RunInfos =
[
// TODO: capture stdout
]
}
let times : TrxReportTimes =
{
Creation = creationTime
Queuing = startTime
Start = startTime
Finish = finishTime
}
let magicGuid = Guid.Parse "13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b"
let results =
results
|> List.collect (fun results -> results.IndividualTestRunMetadata)
|> List.map (fun (i, cause) ->
let exc =
match cause with
| Choice2Of3 _ -> None
| Choice1Of3 (TestMemberFailure.Malformed reasons) ->
{
StackTrace = None
Message = reasons |> String.concat "\n" |> Some
}
|> Some
| Choice1Of3 (TestMemberFailure.Failed fail)
| Choice1Of3 (TestMemberFailure.Failed fail)
| Choice1Of3 (TestMemberFailure.Failed fail) ->
((None, None), fail)
||> List.fold (fun (stackTrace, message) tf ->
match tf with
| TestFailure.TestFailed (UserMethodFailure.Threw (_, exc))
| TestFailure.SetUpFailed (UserMethodFailure.Threw (_, exc))
| TestFailure.TearDownFailed (UserMethodFailure.Threw (_, exc)) ->
let stackTrace =
match stackTrace with
| None -> (exc : Exception).ToString ()
| Some s -> s
(Some stackTrace, message)
| TestFailure.TestFailed (UserMethodFailure.ReturnedNonUnit (_, ret))
| TestFailure.SetUpFailed (UserMethodFailure.ReturnedNonUnit (_, ret))
| TestFailure.TearDownFailed (UserMethodFailure.ReturnedNonUnit (_, ret)) ->
let newMessage = $"returned non-unit value %O{ret}"
let message =
match message with
| None -> newMessage
| Some message -> $"%s{message}\n%s{newMessage}"
(stackTrace, Some message)
)
|> fun (stackTrace, message) ->
{
StackTrace = stackTrace
Message = message
}
|> Some
| Choice3Of3 (UserMethodFailure.Threw (_, exc)) ->
{
StackTrace = (exc : Exception).ToString () |> Some
Message = None
}
|> Some
| Choice3Of3 (UserMethodFailure.ReturnedNonUnit (_, ret)) ->
{
Message = $"returned non-unit value %O{ret}" |> Some
StackTrace = None
}
|> Some
let outcome =
match cause with
| Choice1Of3 _ -> TrxTestOutcome.Failed
| Choice2Of3 TestMemberSuccess.Ok -> TrxTestOutcome.Passed
| Choice2Of3 (TestMemberSuccess.Inconclusive _) -> TrxTestOutcome.Inconclusive
| Choice2Of3 (TestMemberSuccess.Ignored _)
| Choice2Of3 (TestMemberSuccess.Explicit _) -> TrxTestOutcome.NotExecuted
// TODO: we can totally do better here, more fine-grained classification
| Choice3Of3 _ -> TrxTestOutcome.Failed
{
ExecutionId = i.ExecutionId
TestId = i.TestId
TestName = i.TestName
ComputerName = i.ComputerName
Duration = i.End - i.Start
StartTime = i.Start
EndTime = i.End
TestType = magicGuid
Outcome = outcome
TestListId = testListId
RelativeResultsDirectory = i.ExecutionId.ToString () // for some reason
Output =
match i.StdOut, i.StdErr, exc with
| None, None, None -> None
| stdout, stderr, exc ->
Some
{
TrxOutput.StdOut = stdout
StdErr = stderr
ErrorInfo = exc
}
}
)
{
Id = Guid.NewGuid ()
Name = $"@%s{hostname} %s{finishTimeHumanReadable}"
Times = times
Settings = settings
Results = results
TestDefinitions = testDefinitions
TestEntries = testEntries
TestLists = [ testList ]
ResultsSummary = resultSummary
}

View File

@@ -1,3 +1,26 @@
WoofWare.NUnitTestRunner.Args inherit obj, implements WoofWare.NUnitTestRunner.Args System.IEquatable, System.Collections.IStructuralEquatable
WoofWare.NUnitTestRunner.Args..ctor [constructor]: (System.IO.FileInfo, System.IO.FileInfo option, (string * WoofWare.NUnitTestRunner.Filter) option, WoofWare.NUnitTestRunner.LogLevel, int option, System.TimeSpan option)
WoofWare.NUnitTestRunner.Args.Dll [property]: [read-only] System.IO.FileInfo
WoofWare.NUnitTestRunner.Args.Filter [property]: [read-only] (string * WoofWare.NUnitTestRunner.Filter) option
WoofWare.NUnitTestRunner.Args.get_Dll [method]: unit -> System.IO.FileInfo
WoofWare.NUnitTestRunner.Args.get_Filter [method]: unit -> (string * WoofWare.NUnitTestRunner.Filter) option
WoofWare.NUnitTestRunner.Args.get_LevelOfParallelism [method]: unit -> int option
WoofWare.NUnitTestRunner.Args.get_Logging [method]: unit -> WoofWare.NUnitTestRunner.LogLevel
WoofWare.NUnitTestRunner.Args.get_Timeout [method]: unit -> System.TimeSpan option
WoofWare.NUnitTestRunner.Args.get_Trx [method]: unit -> System.IO.FileInfo option
WoofWare.NUnitTestRunner.Args.LevelOfParallelism [property]: [read-only] int option
WoofWare.NUnitTestRunner.Args.Logging [property]: [read-only] WoofWare.NUnitTestRunner.LogLevel
WoofWare.NUnitTestRunner.Args.Parse [static method]: string list -> WoofWare.NUnitTestRunner.Args
WoofWare.NUnitTestRunner.Args.Timeout [property]: [read-only] System.TimeSpan option
WoofWare.NUnitTestRunner.Args.Trx [property]: [read-only] System.IO.FileInfo option
WoofWare.NUnitTestRunner.AssemblyLevelAttributes inherit obj, implements WoofWare.NUnitTestRunner.AssemblyLevelAttributes System.IEquatable, System.Collections.IStructuralEquatable, WoofWare.NUnitTestRunner.AssemblyLevelAttributes System.IComparable, System.IComparable, System.Collections.IStructuralComparable
WoofWare.NUnitTestRunner.AssemblyLevelAttributes..ctor [constructor]: (int option, WoofWare.NUnitTestRunner.AssemblyParallelScope WoofWare.NUnitTestRunner.Parallelizable option)
WoofWare.NUnitTestRunner.AssemblyLevelAttributes.get_Parallelism [method]: unit -> int option
WoofWare.NUnitTestRunner.AssemblyLevelAttributes.get_Parallelizable [method]: unit -> WoofWare.NUnitTestRunner.AssemblyParallelScope WoofWare.NUnitTestRunner.Parallelizable option
WoofWare.NUnitTestRunner.AssemblyLevelAttributes.Parallelism [property]: [read-only] int option
WoofWare.NUnitTestRunner.AssemblyLevelAttributes.Parallelizable [property]: [read-only] WoofWare.NUnitTestRunner.AssemblyParallelScope WoofWare.NUnitTestRunner.Parallelizable option
WoofWare.NUnitTestRunner.AssemblyLevelAttributesModule inherit obj
WoofWare.NUnitTestRunner.AssemblyLevelAttributesModule.get [static method]: System.Reflection.Assembly -> WoofWare.NUnitTestRunner.AssemblyLevelAttributes
WoofWare.NUnitTestRunner.AssemblyParallelScope inherit obj, implements WoofWare.NUnitTestRunner.AssemblyParallelScope System.IEquatable, System.Collections.IStructuralEquatable, WoofWare.NUnitTestRunner.AssemblyParallelScope System.IComparable, System.IComparable, System.Collections.IStructuralComparable - union type with 2 cases
WoofWare.NUnitTestRunner.AssemblyParallelScope+Tags inherit obj
WoofWare.NUnitTestRunner.AssemblyParallelScope+Tags.Children [static field]: int = 0
@@ -12,6 +35,8 @@ WoofWare.NUnitTestRunner.AssemblyParallelScope.get_Tag [method]: unit -> int
WoofWare.NUnitTestRunner.AssemblyParallelScope.IsChildren [property]: [read-only] bool
WoofWare.NUnitTestRunner.AssemblyParallelScope.IsFixtures [property]: [read-only] bool
WoofWare.NUnitTestRunner.AssemblyParallelScope.Tag [property]: [read-only] int
WoofWare.NUnitTestRunner.BuildTrxReport inherit obj
WoofWare.NUnitTestRunner.BuildTrxReport.build [static method]: System.Reflection.Assembly -> System.DateTimeOffset -> System.DateTimeOffset -> WoofWare.NUnitTestRunner.FixtureRunResults list -> WoofWare.NUnitTestRunner.TrxReport
WoofWare.NUnitTestRunner.ClassParallelScope inherit obj, implements WoofWare.NUnitTestRunner.ClassParallelScope System.IEquatable, System.Collections.IStructuralEquatable, WoofWare.NUnitTestRunner.ClassParallelScope System.IComparable, System.IComparable, System.Collections.IStructuralComparable - union type with 4 cases
WoofWare.NUnitTestRunner.ClassParallelScope+Tags inherit obj
WoofWare.NUnitTestRunner.ClassParallelScope+Tags.All [static field]: int = 3
@@ -145,6 +170,20 @@ WoofWare.NUnitTestRunner.ITestProgress.OnTestMemberSkipped [method]: string -> u
WoofWare.NUnitTestRunner.ITestProgress.OnTestMemberStart [method]: string -> unit
WoofWare.NUnitTestRunner.LoadContext inherit System.Runtime.Loader.AssemblyLoadContext
WoofWare.NUnitTestRunner.LoadContext..ctor [constructor]: (System.IO.FileInfo, System.IO.DirectoryInfo list, WoofWare.NUnitTestRunner.TestContexts)
WoofWare.NUnitTestRunner.LogLevel inherit obj, implements WoofWare.NUnitTestRunner.LogLevel System.IEquatable, System.Collections.IStructuralEquatable, WoofWare.NUnitTestRunner.LogLevel System.IComparable, System.IComparable, System.Collections.IStructuralComparable - union type with 2 cases
WoofWare.NUnitTestRunner.LogLevel+Tags inherit obj
WoofWare.NUnitTestRunner.LogLevel+Tags.Nothing [static field]: int = 0
WoofWare.NUnitTestRunner.LogLevel+Tags.Verbose [static field]: int = 1
WoofWare.NUnitTestRunner.LogLevel.get_IsNothing [method]: unit -> bool
WoofWare.NUnitTestRunner.LogLevel.get_IsVerbose [method]: unit -> bool
WoofWare.NUnitTestRunner.LogLevel.get_Nothing [static method]: unit -> WoofWare.NUnitTestRunner.LogLevel
WoofWare.NUnitTestRunner.LogLevel.get_Tag [method]: unit -> int
WoofWare.NUnitTestRunner.LogLevel.get_Verbose [static method]: unit -> WoofWare.NUnitTestRunner.LogLevel
WoofWare.NUnitTestRunner.LogLevel.IsNothing [property]: [read-only] bool
WoofWare.NUnitTestRunner.LogLevel.IsVerbose [property]: [read-only] bool
WoofWare.NUnitTestRunner.LogLevel.Nothing [static property]: [read-only] WoofWare.NUnitTestRunner.LogLevel
WoofWare.NUnitTestRunner.LogLevel.Tag [property]: [read-only] int
WoofWare.NUnitTestRunner.LogLevel.Verbose [static property]: [read-only] WoofWare.NUnitTestRunner.LogLevel
WoofWare.NUnitTestRunner.Match inherit obj, implements WoofWare.NUnitTestRunner.Match System.IEquatable, System.Collections.IStructuralEquatable, WoofWare.NUnitTestRunner.Match System.IComparable, System.IComparable, System.Collections.IStructuralComparable - union type with 2 cases
WoofWare.NUnitTestRunner.Match+Contains inherit WoofWare.NUnitTestRunner.Match
WoofWare.NUnitTestRunner.Match+Contains.get_Item [method]: unit -> string
@@ -204,6 +243,8 @@ WoofWare.NUnitTestRunner.ParallelQueue.Run [method]: WoofWare.NUnitTestRunner.Te
WoofWare.NUnitTestRunner.ParallelQueue.RunTestSetup [method]: WoofWare.NUnitTestRunner.TestFixtureRunningToken -> (unit -> 'a) -> ('a * WoofWare.NUnitTestRunner.TestFixtureSetupToken) System.Threading.Tasks.Task
WoofWare.NUnitTestRunner.ParallelQueue.RunTestTearDown [method]: WoofWare.NUnitTestRunner.TestFixtureSetupToken -> (unit -> 'a) -> ('a * WoofWare.NUnitTestRunner.TestFixtureTearDownToken) System.Threading.Tasks.Task
WoofWare.NUnitTestRunner.ParallelQueue.StartTestFixture [method]: WoofWare.NUnitTestRunner.TestFixture -> WoofWare.NUnitTestRunner.TestFixtureRunningToken System.Threading.Tasks.Task
WoofWare.NUnitTestRunner.SetBaseDir inherit obj, implements IDisposable
WoofWare.NUnitTestRunner.SetBaseDir..ctor [constructor]: System.IO.FileInfo
WoofWare.NUnitTestRunner.SingleTestMethod inherit obj, implements WoofWare.NUnitTestRunner.SingleTestMethod System.IEquatable, System.Collections.IStructuralEquatable
WoofWare.NUnitTestRunner.SingleTestMethod..ctor [constructor]: (System.Reflection.MethodInfo, WoofWare.NUnitTestRunner.TestKind, WoofWare.NUnitTestRunner.Modifier list, string list, int option, WoofWare.NUnitTestRunner.Combinatorial option, unit WoofWare.NUnitTestRunner.Parallelizable option)
WoofWare.NUnitTestRunner.SingleTestMethod.Categories [property]: [read-only] string list

View File

@@ -19,6 +19,7 @@
<ItemGroup>
<Compile Include="AssemblyInfo.fs" />
<Compile Include="AppContext.fs" />
<Compile Include="RuntimeConfig.fs" />
<Compile Include="GeneratedRuntimeConfig.fs">
<MyriadFile>RuntimeConfig.fs</MyriadFile>
@@ -29,6 +30,7 @@
<Compile Include="Result.fs" />
<Compile Include="Domain.fs" />
<Compile Include="Filter.fs" />
<Compile Include="Args.fs" />
<Compile Include="ParallelQueue.fs" />
<Compile Include="SingleTestMethod.fs" />
<Compile Include="TestProgress.fs" />
@@ -36,6 +38,8 @@
<Compile Include="TestFixture.fs" />
<Compile Include="Xml.fs" />
<Compile Include="TrxReport.fs" />
<Compile Include="CreateTrxReport.fs" />
<Compile Include="AssemblyLevelAttributes.fs" />
<None Include="..\README.md">
<Pack>True</Pack>
<PackagePath>\</PackagePath>
@@ -44,14 +48,14 @@
<EmbeddedResource Include="version.json" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="WoofWare.PrattParser" Version="0.2.1" />
<PackageReference Include="WoofWare.PrattParser" Version="0.2.2" />
<PackageReference Update="FSharp.Core" Version="6.0.0" />
<PackageReference Include="WoofWare.DotnetRuntimeLocator" Version="0.1.9" />
<PackageReference Include="WoofWare.Myriad.Plugins.Attributes" Version="3.1.7" />
<PackageReference Include="Myriad.SDK" Version="0.8.3" />
<PackageReference Include="Myriad.SDK" Version="0.8.3" PrivateAssets="all" />
<PackageReference Include="WoofWare.Myriad.Plugins" Version="$(WoofWareMyriadPluginVersion)" PrivateAssets="all" />
</ItemGroup>
<ItemGroup>
<MyriadSdkGenerator Include="$(NuGetPackageRoot)/woofware.myriad.plugins/$(WoofWareMyriadPluginVersion)/lib/net6.0/WoofWare.Myriad.Plugins.dll" />
</ItemGroup>

View File

@@ -1,5 +1,5 @@
{
"version": "0.13",
"version": "0.15",
"publicReleaseRefSpec": [
"^refs/heads/main$"
],
@@ -8,4 +8,4 @@
":/Directory.Build.props",
":/README.md"
]
}
}

View File

@@ -5,100 +5,6 @@ open System.IO
open System.Threading.Tasks
open Spectre.Console
// Fix for https://github.com/Smaug123/unofficial-nunit-runner/issues/8
// Set AppContext.BaseDirectory to where the test DLL is.
// (This tells the DLL loader to look next to the test DLL for dependencies.)
type SetBaseDir (testDll : FileInfo) =
let oldBaseDir = AppContext.BaseDirectory
do AppContext.SetData ("APP_CONTEXT_BASE_DIRECTORY", testDll.Directory.FullName)
interface IDisposable with
member _.Dispose () =
AppContext.SetData ("APP_CONTEXT_BASE_DIRECTORY", oldBaseDir)
[<RequireQualifiedAccess>]
type LogLevel =
| Nothing
| Verbose
[<AutoOpen>]
module Patterns =
let (|Key|_|) (start : string) (s : string) : string option =
if s.StartsWith (start + "=", StringComparison.Ordinal) then
s.Substring (start.Length + 1) |> Some
else
None
type Args =
{
Dll : FileInfo
Trx : FileInfo option
Filter : Filter option
Logging : LogLevel
LevelOfParallelism : int option
Timeout : TimeSpan option
}
static member Parse (args : string list) : Args =
match args with
| [] -> failwith "The first arg must be a positional arg, the DLL to test."
| dll :: args ->
let rec go
(trx : FileInfo option)
(filter : Filter option)
(logging : LogLevel option)
(par : int option)
(timeout : TimeSpan option)
(args : string list)
=
match args with
| [] ->
{
Dll = FileInfo dll
Trx = trx
Filter = filter
Logging = logging |> Option.defaultValue LogLevel.Nothing
LevelOfParallelism = par
Timeout = timeout
}
| Key "--filter" filterStr :: rest
| "--filter" :: filterStr :: rest ->
match filter with
| Some _ -> failwith "Two conflicting filters; you can only specify --filter once"
| None -> go trx (Some (Filter.parse filterStr)) logging par timeout rest
| Key "--trx" trxStr :: rest
| "--trx" :: trxStr :: rest ->
match trx with
| Some _ -> failwith "Two conflicting TRX outputs; you can only specify --trx once"
| None -> go (Some (FileInfo trxStr)) filter logging par timeout rest
| Key "--verbose" verboseStr :: rest
| "--verbose" :: verboseStr :: rest ->
match logging with
| Some _ -> failwith "Two conflicting --verbose outputs; you can only specify --verbose once"
| None ->
let verbose =
if Boolean.Parse verboseStr then
LogLevel.Verbose
else
LogLevel.Nothing
go trx filter (Some verbose) par timeout rest
| Key "--parallelism" parStr :: rest
| "--parallelism" :: parStr :: rest ->
match par with
| Some _ -> failwith "Two conflicting --parallelism outputs; you can only specify --parallelism once"
| None -> go trx filter logging (Some (Int32.Parse parStr)) timeout rest
| Key "--timeout-seconds" timeoutStr :: rest
| "--timeout-seconds" :: timeoutStr :: rest ->
match timeout with
| Some _ ->
failwith "Two conflicting --timeout-seconds outputs; you can only specify --timeout-seconds once"
| None -> go trx filter logging par (Some (TimeSpan.FromSeconds (Int32.Parse timeoutStr |> float))) rest
| k :: _rest -> failwith $"Unrecognised arg %s{k}"
go None None None None None args
module Program =
let main argv =
let startTime = DateTimeOffset.Now
@@ -107,7 +13,7 @@ module Program =
let filter =
match args.Filter with
| Some filter -> Filter.shouldRun filter
| Some (_, filter) -> Filter.shouldRun filter
| None -> fun _ _ -> true
let stderr =
@@ -132,47 +38,10 @@ module Program =
let ctx = LoadContext (args.Dll, runtime, contexts)
let assy = ctx.LoadFromAssemblyPath args.Dll.FullName
let levelOfParallelism, par =
((None, None), assy.CustomAttributes)
||> Seq.fold (fun (levelPar, par) attr ->
match attr.AttributeType.FullName with
| "NUnit.Framework.LevelOfParallelismAttribute" ->
let arg = attr.ConstructorArguments |> Seq.exactlyOne |> _.Value |> unbox<int>
match levelPar with
| None -> (Some arg, par)
| Some existing ->
failwith $"Assembly %s{assy.Location} declares parallelism %i{arg} and also %i{existing}"
| "NUnit.Framework.NonParallelizableAttribute" ->
match levelPar with
| None -> (Some 1, par)
| Some existing ->
failwith
$"Assembly %s{assy.Location} declares non-parallelizable and also parallelism %i{existing}"
| "NUnit.Framework.ParallelizableAttribute" ->
match par with
| Some _ -> failwith "Got multiple Parallelize attributes in assembly"
| None ->
match attr.ConstructorArguments |> Seq.toList with
| [] -> levelPar, Some (Parallelizable.Yes AssemblyParallelScope.Fixtures)
| [ v ] ->
match v.Value with
| :? int as v ->
match v with
| 512 -> levelPar, Some (Parallelizable.Yes AssemblyParallelScope.Fixtures)
| 256 -> levelPar, Some (Parallelizable.Yes AssemblyParallelScope.Children)
| 257 ->
failwith "ParallelScope.All is invalid on assemblies; only Fixtures or Children"
| 1 ->
failwith "ParallelScope.Self is invalid on assemblies; only Fixtures or Children"
| v -> failwith $"Could not recognise value %i{v} of parallel scope on assembly"
| v -> failwith $"Unexpectedly non-int value %O{v} of parallel scope on assembly"
| _ -> failwith "unexpectedly got multiple args to Parallelizable on assembly"
| _ -> levelPar, par
)
let attrs = AssemblyLevelAttributes.get assy
let levelOfParallelism =
match args.LevelOfParallelism, levelOfParallelism with
match args.LevelOfParallelism, attrs.Parallelism with
| None, None -> None
| Some taken, Some ignored ->
match args.Logging with
@@ -187,7 +56,7 @@ module Program =
let testFixtures = assy.ExportedTypes |> Seq.map TestFixture.parse |> Seq.toList
use par = new ParallelQueue (levelOfParallelism, par)
use par = new ParallelQueue (levelOfParallelism, attrs.Parallelizable)
let creationTime = DateTimeOffset.Now
@@ -206,233 +75,7 @@ module Program =
let results = results.Result |> Seq.concat |> List.ofSeq
let finishTime = DateTimeOffset.Now
let finishTimeHumanReadable = finishTime.ToString @"yyyy-MM-dd HH:mm:ss"
let nowMachine = finishTime.ToString @"yyyy-MM-dd_HH_mm_ss"
let testListId = Guid.NewGuid ()
let testDefinitions, testEntries =
results
|> List.collect (fun results -> results.IndividualTestRunMetadata)
|> List.map (fun (data, _) ->
let defn =
{
Name = data.TestName
Storage = assy.Location.ToLowerInvariant ()
Id = data.TestId
Execution =
{
Id = data.ExecutionId
}
TestMethod =
{
CodeBase = assy.Location
AdapterTypeName = Uri "executor://woofware/"
ClassName = data.ClassName
Name = data.TestName
}
}
let entry : TrxTestEntry =
{
TestListId = testListId
ExecutionId = data.ExecutionId
TestId = data.TestId
}
defn, entry
)
|> List.unzip
let hostname = Environment.MachineName
let settings =
{
Name = "default"
Id = Guid.NewGuid ()
Deployment =
{
RunDeploymentRoot = $"_%s{hostname}_%s{nowMachine}"
}
}
let testList : TrxTestListEntry =
{
Id = testListId
Name = "All"
}
let counters =
(TrxCounters.Zero, results)
// TODO: this is woefully inefficient
||> List.fold (fun counters results ->
let counters =
(counters, results.Failed)
||> List.fold (fun counters (_, _) ->
// TODO: the counters can be more specific about the failure mode
counters.AddFailed ()
)
let counters =
(counters, results.OtherFailures)
||> List.fold (fun counters _ ->
// TODO: the counters can be more specific about the failure mode
counters.AddFailed ()
)
(counters, results.Success)
||> List.fold (fun counters (_, success, _) ->
match success with
| TestMemberSuccess.Ok -> counters.AddPassed ()
| TestMemberSuccess.Ignored _
| TestMemberSuccess.Explicit _ -> counters.AddNotExecuted ()
| TestMemberSuccess.Inconclusive _ -> counters.AddInconclusive ()
)
)
// TODO: I'm sure we can do better than this; there's a whole range of possible
// states!
let outcome =
if counters.Failed > 0u then
TrxOutcome.Failed
else
TrxOutcome.Completed
let resultSummary : TrxResultsSummary =
{
Outcome = outcome
Counters = counters
Output =
{
StdOut = None
StdErr = None
ErrorInfo = None
}
RunInfos =
[
// TODO: capture stdout
]
}
let times : TrxReportTimes =
{
Creation = creationTime
Queuing = startTime
Start = startTime
Finish = finishTime
}
let magicGuid = Guid.Parse "13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b"
let results =
results
|> List.collect (fun results -> results.IndividualTestRunMetadata)
|> List.map (fun (i, cause) ->
let exc =
match cause with
| Choice2Of3 _ -> None
| Choice1Of3 (TestMemberFailure.Malformed reasons) ->
{
StackTrace = None
Message = reasons |> String.concat "\n" |> Some
}
|> Some
| Choice1Of3 (TestMemberFailure.Failed fail)
| Choice1Of3 (TestMemberFailure.Failed fail)
| Choice1Of3 (TestMemberFailure.Failed fail) ->
((None, None), fail)
||> List.fold (fun (stackTrace, message) tf ->
match tf with
| TestFailure.TestFailed (UserMethodFailure.Threw (_, exc))
| TestFailure.SetUpFailed (UserMethodFailure.Threw (_, exc))
| TestFailure.TearDownFailed (UserMethodFailure.Threw (_, exc)) ->
let stackTrace =
match stackTrace with
| None -> (exc : Exception).ToString ()
| Some s -> s
(Some stackTrace, message)
| TestFailure.TestFailed (UserMethodFailure.ReturnedNonUnit (_, ret))
| TestFailure.SetUpFailed (UserMethodFailure.ReturnedNonUnit (_, ret))
| TestFailure.TearDownFailed (UserMethodFailure.ReturnedNonUnit (_, ret)) ->
let newMessage = $"returned non-unit value %O{ret}"
let message =
match message with
| None -> newMessage
| Some message -> $"%s{message}\n%s{newMessage}"
(stackTrace, Some message)
)
|> fun (stackTrace, message) ->
{
StackTrace = stackTrace
Message = message
}
|> Some
| Choice3Of3 (UserMethodFailure.Threw (_, exc)) ->
{
StackTrace = (exc : Exception).ToString () |> Some
Message = None
}
|> Some
| Choice3Of3 (UserMethodFailure.ReturnedNonUnit (_, ret)) ->
{
Message = $"returned non-unit value %O{ret}" |> Some
StackTrace = None
}
|> Some
let outcome =
match cause with
| Choice1Of3 _ -> TrxTestOutcome.Failed
| Choice2Of3 TestMemberSuccess.Ok -> TrxTestOutcome.Passed
| Choice2Of3 (TestMemberSuccess.Inconclusive _) -> TrxTestOutcome.Inconclusive
| Choice2Of3 (TestMemberSuccess.Ignored _)
| Choice2Of3 (TestMemberSuccess.Explicit _) -> TrxTestOutcome.NotExecuted
// TODO: we can totally do better here, more fine-grained classification
| Choice3Of3 _ -> TrxTestOutcome.Failed
{
ExecutionId = i.ExecutionId
TestId = i.TestId
TestName = i.TestName
ComputerName = i.ComputerName
Duration = i.End - i.Start
StartTime = i.Start
EndTime = i.End
TestType = magicGuid
Outcome = outcome
TestListId = testListId
RelativeResultsDirectory = i.ExecutionId.ToString () // for some reason
Output =
match i.StdOut, i.StdErr, exc with
| None, None, None -> None
| stdout, stderr, exc ->
Some
{
TrxOutput.StdOut = stdout
StdErr = stderr
ErrorInfo = exc
}
}
)
let report : TrxReport =
{
Id = Guid.NewGuid ()
Name = $"@%s{hostname} %s{finishTimeHumanReadable}"
Times = times
Settings = settings
Results = results
TestDefinitions = testDefinitions
TestEntries = testEntries
TestLists = [ testList ]
ResultsSummary = resultSummary
}
let report = BuildTrxReport.build assy creationTime startTime results
match args.Trx with
| Some trxPath ->
@@ -442,7 +85,7 @@ module Program =
Console.Error.WriteLine $"Written TRX file: %s{trxPath.FullName}"
| None -> ()
match outcome with
match report.ResultsSummary.Outcome with
| TrxOutcome.Completed -> 0
| _ -> 1

View File

@@ -57,8 +57,8 @@
};
};
devShell = pkgs.mkShell {
buildInputs = [dotnet-sdk];
packages = [
(pkgs.dotnetCorePackages.combinePackages [dotnet-sdk pkgs.dotnetCorePackages.runtime_6_0])
pkgs.alejandra
pkgs.nodePackages.markdown-link-check
pkgs.shellcheck

View File

@@ -263,7 +263,7 @@
})
(fetchNuGet {
pname = "WoofWare.PrattParser";
version = "0.2.1";
sha256 = "1cb9496fbbrdc40dirjmc7ax02ghr27ahqq5hpk96rdzyaang9hg";
version = "0.2.2";
sha256 = "0cgrmd1kc3k224lsjhy5npalwg6kpqd8nx78szi1yq67kyb0farq";
})
]