Prepare for release (#27)

This commit is contained in:
Patrick Stevens
2024-06-05 23:21:45 +01:00
committed by GitHub
parent f6e907395c
commit 58b1dfedfd
17 changed files with 718 additions and 156 deletions

View File

@@ -88,12 +88,12 @@ module TestValues =
let ``Values are all OK`` () =
seen1 |> Seq.toList |> shouldEqual [ true ; false ]
seen2 |> Seq.toList |> shouldEqual [ (true, false) ; (false, true) ]
seen3 |> Seq.toList |> shouldEqual [ (88, 29) ; (31, 0) ]
seen4 |> Seq.toList |> shouldEqual [ ("hi", "ohh") ; ("bye", null) ]
seen5 |> Seq.toList |> shouldEqual [ (88, 29) ; (31, 29) ]
seen6 |> Seq.toList |> shouldEqual [ ("hi", "ohh") ; ("bye", "ohh") ]
seen7 |> Seq.toList |> shouldEqual [ (88, 29) ; (31, 29) ]
seen8 |> Seq.toList |> shouldEqual [ ("hi", "ohh") ; ("bye", "ohh") ]
seen3 |> Seq.toList |> shouldEqual [ (88, box 29) ; (31, box 0) ]
seen4 |> Seq.toList |> shouldEqual [ ("hi", box "ohh") ; ("bye", null) ]
seen5 |> Seq.toList |> shouldEqual [ (88, box 29) ; (31, box 29) ]
seen6 |> Seq.toList |> shouldEqual [ ("hi", box "ohh") ; ("bye", box "ohh") ]
seen7 |> Seq.toList |> shouldEqual [ (88, box 29) ; (31, box 29) ]
seen8 |> Seq.toList |> shouldEqual [ ("hi", box "ohh") ; ("bye", box "ohh") ]
seen9
|> Seq.toList

15
Directory.Build.props Normal file
View File

@@ -0,0 +1,15 @@
<Project>
<PropertyGroup>
<DebugType Condition=" '$(DebugType)' == '' ">embedded</DebugType>
<Deterministic>true</Deterministic>
<NetCoreTargetingPackRoot>[UNDEFINED]</NetCoreTargetingPackRoot>
<DisableImplicitLibraryPacksFolder>true</DisableImplicitLibraryPacksFolder>
<DisableImplicitNuGetFallbackFolder>true</DisableImplicitNuGetFallbackFolder>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<DebugType>embedded</DebugType>
<WarnOn>FS3388,FS3559</WarnOn>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Nerdbank.GitVersioning" Version="3.6.139" PrivateAssets="all"/>
</ItemGroup>
</Project>

View File

@@ -0,0 +1,7 @@
namespace TestRunner.AssemblyInfo
open System.Runtime.CompilerServices
[<assembly : InternalsVisibleTo("TestRunner.Test")>]
do ()

View File

@@ -2,43 +2,84 @@ namespace TestRunner
open System.Reflection
/// A modifier on whether a given test should be run.
[<RequireQualifiedAccess>]
type Modifier =
/// This test is Explicit: it can only be run by an explicit instruction to do so.
/// (As of this writing, the console runner will never run such tests.)
| Explicit of reason : string option
/// This test is Ignored: it will never be run by the harness.
| Ignored of reason : string option
/// Describes where data comes from, if any, to provide the args to this test.
[<RequireQualifiedAccess>]
type TestKind =
/// This test takes no arguments.
| Single
/// This test has arguments supplied by TestCaseSource (i.e. we look for members with the given names, and
/// populate the args from those).
| Source of string list
/// This test has arguments supplied by TestCase attributes.
| Data of obj list list
/// Determines whether a set of `[<Value>]`s will be combined elementwise or Cartesian-product-wise.
type Combinatorial =
/// Combine `[<Value>]`s to produce every possible combination of args drawn from the respective sets.
| Combinatorial
/// Combine `[<Value>]`s such that one test is "the first Value from each", one test is "the second Value from
/// each", and so on. Spare slots are filled with `Unchecked.defaultof<_>`.
| Sequential
/// A single method or member which holds some tests. (Often such a member will represent only one test, but e.g.
/// if it has [<TestCaseSource>] then it represents multiple tests.)
type SingleTestMethod =
{
/// The method which we need to invoke, possibly some args, to run the test.
Method : MethodInfo
/// Where the data comes from to populate the args for this method.
Kind : TestKind
/// Any statements about whether the runner should run this test.
/// (This does not include use of `--filter`s.)
Modifiers : Modifier list
/// `[<Category>]`s this test is in.
Categories : string list
/// Whether we should run this test repeatedly, and if so, how many times.
Repeat : int option
/// If this test has data supplied by `[<Value>]` annotations, specifies how those annotations are combined
/// to produce the complete collection of args.
Combinatorial : Combinatorial option
}
/// Human-readable name of this test method.
member this.Name = this.Method.Name
/// A test fixture (usually represented by the [<TestFixture>]` attribute), which may contain many tests,
/// each of which may run many times.
type TestFixture =
{
/// Fully-qualified name of this fixture (e.g. MyThing.Test.Foo for `[<TestFixture>] module Foo` in the
/// `MyThing.Test` assembly).
Name : string
/// A method which is run once when this test fixture starts, before any other setup logic and before
/// any tests run. If this method fails, no tests will run and no per-test setup/teardown logic will run,
/// but OneTimeTearDown will run.
OneTimeSetUp : MethodInfo option
/// A method which is run once, after any other tear-down logic and after all tests run, even if everything
/// else failed before this (i.e. even if OneTimeSetUp failed, even if all tests failed, etc).
OneTimeTearDown : MethodInfo option
/// Methods which are run in some arbitrary order before each individual test. If any of these fail, the test
/// will not run, but the TearDown methods will still run, and OneTimeTearDown will still run at the end of
/// the fixture. If the first SetUp we run fails, we don't define whether the other SetUps run before
/// we proceed directly to running all the TearDowns.
SetUp : MethodInfo list
/// Methods which are run in some arbitrary order after each individual test, even if the test or its setup
/// failed. If the first TearDown we run fails, we don't define whether the other TearDowns run.
TearDown : MethodInfo list
/// The individual test methods present within this fixture.
Tests : SingleTestMethod list
}
/// A test fixture about which we know nothing. No tests, no setup/teardown.
static member Empty (name : string) =
{
Name = name
@@ -49,11 +90,15 @@ type TestFixture =
Tests = []
}
/// User code in the unit under test has failed somehow.
[<RequireQualifiedAccess>]
type UserMethodFailure =
/// A method ran to completion and returned a value, when it was expected to return nothing.
| ReturnedNonUnit of name : string * result : obj
/// A method threw.
| Threw of name : string * exn
/// Human-readable representation of the user failure.
override this.ToString () =
match this with
| UserMethodFailure.ReturnedNonUnit (method, ret) ->
@@ -61,8 +106,28 @@ type UserMethodFailure =
| UserMethodFailure.Threw (method, exc) ->
$"User-defined method %s{method} threw: %s{exc.Message}\n %s{exc.StackTrace}"
/// Name (not fully-qualified) of the method which failed.
member this.Name =
match this with
| UserMethodFailure.Threw (name, _)
| UserMethodFailure.ReturnedNonUnit (name, _) -> name
/// Represents the failure of a single run of one test. An error signalled this way is a user error: the unit under
/// test has misbehaved.
[<RequireQualifiedAccess>]
type TestFailure =
/// The test itself failed. (Setup must have succeeded if you get this.)
| TestFailed of UserMethodFailure
/// We failed to set up the test (e.g. its SetUp failed). If this happens, we won't proceed
/// to running the test or running any TearDown for that test.
| SetUpFailed of UserMethodFailure
/// We failed to tear down the test (e.g. its TearDown failed). This can happen even if the test failed,
/// because we always run tear-downs, even after failed tests.
| TearDownFailed of UserMethodFailure
/// Name (not fully-qualified) of the method which failed.
member this.Name =
match this with
| TestFailure.TestFailed f
| TestFailure.SetUpFailed f
| TestFailure.TearDownFailed f -> f.Name

View File

@@ -7,19 +7,19 @@ open PrattParser
// https://learn.microsoft.com/en-us/dotnet/core/testing/selective-unit-tests?pivots=mstest
[<RequireQualifiedAccess>]
type FilterIntermediate =
type internal ParsedFilter =
| FullyQualifiedName
| Name
| TestCategory
| Not of FilterIntermediate
| Or of FilterIntermediate * FilterIntermediate
| And of FilterIntermediate * FilterIntermediate
| Equal of FilterIntermediate * FilterIntermediate
| Contains of FilterIntermediate * FilterIntermediate
| Not of ParsedFilter
| Or of ParsedFilter * ParsedFilter
| And of ParsedFilter * ParsedFilter
| Equal of ParsedFilter * ParsedFilter
| Contains of ParsedFilter * ParsedFilter
| String of string
[<RequireQualifiedAccess>]
type TokenType =
type internal TokenType =
| FullyQualifiedName
| Name
| TestCategory
@@ -34,14 +34,14 @@ type TokenType =
| NotContains
| String
type Token =
type internal Token =
{
Type : TokenType
Trivia : int * int
}
[<RequireQualifiedAccess>]
module Token =
module internal Token =
let inline standalone (ty : TokenType) (charPos : int) : Token =
{
Type = ty
@@ -66,7 +66,7 @@ module Token =
| _ -> None
[<RequireQualifiedAccess>]
module Lexer =
module internal Lexer =
let lex (s : string) : Token seq =
seq {
let mutable i = 0
@@ -121,15 +121,15 @@ module Lexer =
}
[<RequireQualifiedAccess>]
module FilterIntermediate =
let private atom (inputString : string) (token : Token) : FilterIntermediate option =
module internal ParsedFilter =
let private atom (inputString : string) (token : Token) : ParsedFilter option =
let start, len = token.Trivia
match token.Type with
| TokenType.String -> Some (FilterIntermediate.String (inputString.Substring (start, len)))
| TokenType.FullyQualifiedName -> Some FilterIntermediate.FullyQualifiedName
| TokenType.Name -> Some FilterIntermediate.Name
| TokenType.TestCategory -> Some FilterIntermediate.TestCategory
| TokenType.String -> Some (ParsedFilter.String (inputString.Substring (start, len)))
| TokenType.FullyQualifiedName -> Some ParsedFilter.FullyQualifiedName
| TokenType.Name -> Some ParsedFilter.Name
| TokenType.TestCategory -> Some ParsedFilter.TestCategory
| TokenType.OpenParen -> None
| TokenType.CloseParen -> None
| TokenType.And -> None
@@ -141,20 +141,14 @@ module FilterIntermediate =
| TokenType.Contains -> None
let parser =
Parser.make<_, Token, FilterIntermediate> _.Type atom
|> Parser.withInfix TokenType.And (10, 11) (fun a b -> FilterIntermediate.And (a, b))
|> Parser.withInfix TokenType.Equal (15, 16) (fun a b -> FilterIntermediate.Equal (a, b))
|> Parser.withInfix
TokenType.NotEqual
(15, 16)
(fun a b -> FilterIntermediate.Not (FilterIntermediate.Equal (a, b)))
|> Parser.withInfix TokenType.Contains (15, 16) (fun a b -> FilterIntermediate.Contains (a, b))
|> Parser.withInfix
TokenType.NotContains
(15, 16)
(fun a b -> FilterIntermediate.Not (FilterIntermediate.Contains (a, b)))
|> Parser.withInfix TokenType.Or (5, 6) (fun a b -> FilterIntermediate.Or (a, b))
|> Parser.withUnaryPrefix TokenType.Not ((), 13) FilterIntermediate.Not
Parser.make<_, Token, ParsedFilter> _.Type atom
|> Parser.withInfix TokenType.And (10, 11) (fun a b -> ParsedFilter.And (a, b))
|> Parser.withInfix TokenType.Equal (15, 16) (fun a b -> ParsedFilter.Equal (a, b))
|> Parser.withInfix TokenType.NotEqual (15, 16) (fun a b -> ParsedFilter.Not (ParsedFilter.Equal (a, b)))
|> Parser.withInfix TokenType.Contains (15, 16) (fun a b -> ParsedFilter.Contains (a, b))
|> Parser.withInfix TokenType.NotContains (15, 16) (fun a b -> ParsedFilter.Not (ParsedFilter.Contains (a, b)))
|> Parser.withInfix TokenType.Or (5, 6) (fun a b -> ParsedFilter.Or (a, b))
|> Parser.withUnaryPrefix TokenType.Not ((), 13) ParsedFilter.Not
|> Parser.withBracketLike
TokenType.OpenParen
{
@@ -164,65 +158,102 @@ module FilterIntermediate =
Construct = List.exactlyOne
}
let parse (s : string) : FilterIntermediate =
let parse (s : string) : ParsedFilter =
let parsed, remaining = Parser.execute parser s (Lexer.lex s |> Seq.toList)
if not remaining.IsEmpty then
failwith $"Leftover tokens: %O{remaining}"
match parsed with
| FilterIntermediate.String _ -> FilterIntermediate.Contains (FilterIntermediate.FullyQualifiedName, parsed)
| ParsedFilter.String _ -> ParsedFilter.Contains (ParsedFilter.FullyQualifiedName, parsed)
| _ -> parsed
/// The type of matching which this filter will perform.
type Match =
/// This filter will only match if its argument is exactly (case-sensitively) equal to this.
| Exact of string
/// This filter will match if its argument (case-sensitively) contains this substring.
| Contains of string
/// A filter which is to be applied when running tests, to determine which tests to run.
[<RequireQualifiedAccess>]
type Filter =
/// The fully qualified name of the test must match this.
| FullyQualifiedName of Match
/// The name (without its assembly prepended) of the test must match this.
| Name of Match
/// The test must be in a matching category.
| TestCategory of Match
/// The test must not match this filter.
| Not of Filter
/// The test must match at least one of these filters.
| Or of Filter * Filter
/// The test must match both of these filters.
| And of Filter * Filter
/// Methods for manipulating filters.
[<RequireQualifiedAccess>]
module Filter =
let private unescape (s : string) : string =
// TODO: XML escaping
s
let rec make (fi : FilterIntermediate) : Filter =
let rec internal makeParsed (fi : ParsedFilter) : Filter =
match fi with
| FilterIntermediate.Not x -> Filter.Not (make x)
| FilterIntermediate.FullyQualifiedName -> failwith "malformed filter: found FullyQualifiedName with no operand"
| FilterIntermediate.Name -> failwith "malformed filter: found Name with no operand"
| FilterIntermediate.TestCategory -> failwith "malformed filter: found TestCategory with no operand"
| FilterIntermediate.Or (a, b) -> Filter.Or (make a, make b)
| FilterIntermediate.And (a, b) -> Filter.And (make a, make b)
| FilterIntermediate.Equal (key, value) ->
| ParsedFilter.Not x -> Filter.Not (makeParsed x)
| ParsedFilter.FullyQualifiedName -> failwith "malformed filter: found FullyQualifiedName with no operand"
| ParsedFilter.Name -> failwith "malformed filter: found Name with no operand"
| ParsedFilter.TestCategory -> failwith "malformed filter: found TestCategory with no operand"
| ParsedFilter.Or (a, b) -> Filter.Or (makeParsed a, makeParsed b)
| ParsedFilter.And (a, b) -> Filter.And (makeParsed a, makeParsed b)
| ParsedFilter.Equal (key, value) ->
let value =
match value with
| FilterIntermediate.String s -> unescape s
| ParsedFilter.String s -> unescape s
| _ -> failwith $"malformed filter: found non-string operand on RHS of equality, '%O{value}'"
match key with
| FilterIntermediate.TestCategory -> Filter.TestCategory (Match.Exact value)
| FilterIntermediate.FullyQualifiedName -> Filter.FullyQualifiedName (Match.Exact value)
| FilterIntermediate.Name -> Filter.Name (Match.Exact value)
| ParsedFilter.TestCategory -> Filter.TestCategory (Match.Exact value)
| ParsedFilter.FullyQualifiedName -> Filter.FullyQualifiedName (Match.Exact value)
| ParsedFilter.Name -> Filter.Name (Match.Exact value)
| _ -> failwith $"Malformed filter: left-hand side of Equals clause must be e.g. TestCategory, was %O{key}"
| FilterIntermediate.Contains (key, value) ->
| ParsedFilter.Contains (key, value) ->
let value =
match value with
| FilterIntermediate.String s -> unescape s
| ParsedFilter.String s -> unescape s
| _ -> failwith $"malformed filter: found non-string operand on RHS of containment, '%O{value}'"
match key with
| FilterIntermediate.TestCategory -> Filter.TestCategory (Match.Contains value)
| FilterIntermediate.FullyQualifiedName -> Filter.FullyQualifiedName (Match.Contains value)
| FilterIntermediate.Name -> Filter.Name (Match.Contains value)
| ParsedFilter.TestCategory -> Filter.TestCategory (Match.Contains value)
| ParsedFilter.FullyQualifiedName -> Filter.FullyQualifiedName (Match.Contains value)
| ParsedFilter.Name -> Filter.Name (Match.Contains value)
| _ ->
failwith $"Malformed filter: left-hand side of Contains clause must be e.g. TestCategory, was %O{key}"
| FilterIntermediate.String s ->
failwith $"Malformed filter: got verbatim string %s{s} when expected an operation"
| ParsedFilter.String s -> failwith $"Malformed filter: got verbatim string %s{s} when expected an operation"
/// Parse the input string, e.g. the `foo` one might get from `dotnet test --filter foo`.
/// Verbatim strings are assumed to be XML-escaped.
let parse (s : string) : Filter = ParsedFilter.parse s |> makeParsed
/// Convert the representation of a test filter into a function that decides whether to run any given test.
let rec shouldRun (filter : Filter) : TestFixture -> SingleTestMethod -> bool =
match filter with
| Filter.Not filter ->
let inner = shouldRun filter
fun a b -> not (inner a b)
| Filter.And (a, b) ->
let inner1 = shouldRun a
let inner2 = shouldRun b
fun a b -> inner1 a b && inner2 a b
| Filter.Or (a, b) ->
let inner1 = shouldRun a
let inner2 = shouldRun b
fun a b -> inner1 a b || inner2 a b
| Filter.Name (Match.Exact m) -> fun _fixture method -> method.Method.Name = m
| Filter.Name (Match.Contains m) -> fun _fixture method -> method.Method.Name.Contains m
| Filter.FullyQualifiedName (Match.Exact m) -> fun fixture method -> (fixture.Name + method.Method.Name) = m
| Filter.FullyQualifiedName (Match.Contains m) ->
fun fixture method -> (fixture.Name + method.Method.Name).Contains m
| Filter.TestCategory (Match.Contains m) ->
fun _fixture method -> method.Categories |> List.exists (fun cat -> cat.Contains m)
| Filter.TestCategory (Match.Exact m) -> fun _fixture method -> method.Categories |> List.contains m

View File

@@ -1,7 +1,7 @@
namespace TestRunner
[<RequireQualifiedAccess>]
module List =
module internal List =
/// Given e.g. [[1,2],[4,5,6]], returns:
/// [1;4] ; [1;5] ; [1;6] ; [2;4] ; [2;5] ; [2;6]

View File

@@ -3,9 +3,15 @@ namespace TestRunner
open System
open System.Reflection
/// A single method or member which holds some tests. (Often such a member will represent only one test, but e.g.
/// if it has [<TestCaseSource>] then it represents multiple tests.)
[<RequireQualifiedAccess>]
[<CompilationRepresentation(CompilationRepresentationFlags.ModuleSuffix)>]
module SingleTestMethod =
/// Extract a SingleTestMethod from the given MethodInfo that we think represents a test.
/// You pass us the attributes you still haven't parsed from this MethodInfo, and we give you back the sub-list
/// of attributes we were also unable to interpret.
/// You also give us the list of categories with which the containing TestFixture is tagged.
let parse
(parentCategories : string list)
(method : MethodInfo)

View File

@@ -0,0 +1,262 @@
TestRunner.Combinatorial inherit obj, implements TestRunner.Combinatorial System.IEquatable, System.Collections.IStructuralEquatable, TestRunner.Combinatorial System.IComparable, System.IComparable, System.Collections.IStructuralComparable - union type with 2 cases
TestRunner.Combinatorial+Tags inherit obj
TestRunner.Combinatorial+Tags.Combinatorial [static field]: int = 0
TestRunner.Combinatorial+Tags.Sequential [static field]: int = 1
TestRunner.Combinatorial.Combinatorial [static property]: [read-only] TestRunner.Combinatorial
TestRunner.Combinatorial.get_Combinatorial [static method]: unit -> TestRunner.Combinatorial
TestRunner.Combinatorial.get_IsCombinatorial [method]: unit -> bool
TestRunner.Combinatorial.get_IsSequential [method]: unit -> bool
TestRunner.Combinatorial.get_Sequential [static method]: unit -> TestRunner.Combinatorial
TestRunner.Combinatorial.get_Tag [method]: unit -> int
TestRunner.Combinatorial.IsCombinatorial [property]: [read-only] bool
TestRunner.Combinatorial.IsSequential [property]: [read-only] bool
TestRunner.Combinatorial.Sequential [static property]: [read-only] TestRunner.Combinatorial
TestRunner.Combinatorial.Tag [property]: [read-only] int
TestRunner.Filter inherit obj, implements TestRunner.Filter System.IEquatable, System.Collections.IStructuralEquatable, TestRunner.Filter System.IComparable, System.IComparable, System.Collections.IStructuralComparable - union type with 6 cases
TestRunner.Filter+And inherit TestRunner.Filter
TestRunner.Filter+And.get_Item1 [method]: unit -> TestRunner.Filter
TestRunner.Filter+And.get_Item2 [method]: unit -> TestRunner.Filter
TestRunner.Filter+And.Item1 [property]: [read-only] TestRunner.Filter
TestRunner.Filter+And.Item2 [property]: [read-only] TestRunner.Filter
TestRunner.Filter+FullyQualifiedName inherit TestRunner.Filter
TestRunner.Filter+FullyQualifiedName.get_Item [method]: unit -> TestRunner.Match
TestRunner.Filter+FullyQualifiedName.Item [property]: [read-only] TestRunner.Match
TestRunner.Filter+Name inherit TestRunner.Filter
TestRunner.Filter+Name.get_Item [method]: unit -> TestRunner.Match
TestRunner.Filter+Name.Item [property]: [read-only] TestRunner.Match
TestRunner.Filter+Not inherit TestRunner.Filter
TestRunner.Filter+Not.get_Item [method]: unit -> TestRunner.Filter
TestRunner.Filter+Not.Item [property]: [read-only] TestRunner.Filter
TestRunner.Filter+Or inherit TestRunner.Filter
TestRunner.Filter+Or.get_Item1 [method]: unit -> TestRunner.Filter
TestRunner.Filter+Or.get_Item2 [method]: unit -> TestRunner.Filter
TestRunner.Filter+Or.Item1 [property]: [read-only] TestRunner.Filter
TestRunner.Filter+Or.Item2 [property]: [read-only] TestRunner.Filter
TestRunner.Filter+Tags inherit obj
TestRunner.Filter+Tags.And [static field]: int = 5
TestRunner.Filter+Tags.FullyQualifiedName [static field]: int = 0
TestRunner.Filter+Tags.Name [static field]: int = 1
TestRunner.Filter+Tags.Not [static field]: int = 3
TestRunner.Filter+Tags.Or [static field]: int = 4
TestRunner.Filter+Tags.TestCategory [static field]: int = 2
TestRunner.Filter+TestCategory inherit TestRunner.Filter
TestRunner.Filter+TestCategory.get_Item [method]: unit -> TestRunner.Match
TestRunner.Filter+TestCategory.Item [property]: [read-only] TestRunner.Match
TestRunner.Filter.get_IsAnd [method]: unit -> bool
TestRunner.Filter.get_IsFullyQualifiedName [method]: unit -> bool
TestRunner.Filter.get_IsName [method]: unit -> bool
TestRunner.Filter.get_IsNot [method]: unit -> bool
TestRunner.Filter.get_IsOr [method]: unit -> bool
TestRunner.Filter.get_IsTestCategory [method]: unit -> bool
TestRunner.Filter.get_Tag [method]: unit -> int
TestRunner.Filter.IsAnd [property]: [read-only] bool
TestRunner.Filter.IsFullyQualifiedName [property]: [read-only] bool
TestRunner.Filter.IsName [property]: [read-only] bool
TestRunner.Filter.IsNot [property]: [read-only] bool
TestRunner.Filter.IsOr [property]: [read-only] bool
TestRunner.Filter.IsTestCategory [property]: [read-only] bool
TestRunner.Filter.NewAnd [static method]: (TestRunner.Filter, TestRunner.Filter) -> TestRunner.Filter
TestRunner.Filter.NewFullyQualifiedName [static method]: TestRunner.Match -> TestRunner.Filter
TestRunner.Filter.NewName [static method]: TestRunner.Match -> TestRunner.Filter
TestRunner.Filter.NewNot [static method]: TestRunner.Filter -> TestRunner.Filter
TestRunner.Filter.NewOr [static method]: (TestRunner.Filter, TestRunner.Filter) -> TestRunner.Filter
TestRunner.Filter.NewTestCategory [static method]: TestRunner.Match -> TestRunner.Filter
TestRunner.Filter.Tag [property]: [read-only] int
TestRunner.FilterModule inherit obj
TestRunner.FilterModule.parse [static method]: string -> TestRunner.Filter
TestRunner.FilterModule.shouldRun [static method]: TestRunner.Filter -> (TestRunner.TestFixture -> TestRunner.SingleTestMethod -> bool)
TestRunner.FixtureRunResults inherit obj, implements TestRunner.FixtureRunResults System.IEquatable, System.Collections.IStructuralEquatable
TestRunner.FixtureRunResults..ctor [constructor]: (TestRunner.TestMemberFailure list, int, TestRunner.UserMethodFailure list)
TestRunner.FixtureRunResults.Failed [property]: [read-only] TestRunner.TestMemberFailure list
TestRunner.FixtureRunResults.get_Failed [method]: unit -> TestRunner.TestMemberFailure list
TestRunner.FixtureRunResults.get_OtherFailures [method]: unit -> TestRunner.UserMethodFailure list
TestRunner.FixtureRunResults.get_SuccessCount [method]: unit -> int
TestRunner.FixtureRunResults.OtherFailures [property]: [read-only] TestRunner.UserMethodFailure list
TestRunner.FixtureRunResults.SuccessCount [property]: [read-only] int
TestRunner.Match inherit obj, implements TestRunner.Match System.IEquatable, System.Collections.IStructuralEquatable, TestRunner.Match System.IComparable, System.IComparable, System.Collections.IStructuralComparable - union type with 2 cases
TestRunner.Match+Contains inherit TestRunner.Match
TestRunner.Match+Contains.get_Item [method]: unit -> string
TestRunner.Match+Contains.Item [property]: [read-only] string
TestRunner.Match+Exact inherit TestRunner.Match
TestRunner.Match+Exact.get_Item [method]: unit -> string
TestRunner.Match+Exact.Item [property]: [read-only] string
TestRunner.Match+Tags inherit obj
TestRunner.Match+Tags.Contains [static field]: int = 1
TestRunner.Match+Tags.Exact [static field]: int = 0
TestRunner.Match.get_IsContains [method]: unit -> bool
TestRunner.Match.get_IsExact [method]: unit -> bool
TestRunner.Match.get_Tag [method]: unit -> int
TestRunner.Match.IsContains [property]: [read-only] bool
TestRunner.Match.IsExact [property]: [read-only] bool
TestRunner.Match.NewContains [static method]: string -> TestRunner.Match
TestRunner.Match.NewExact [static method]: string -> TestRunner.Match
TestRunner.Match.Tag [property]: [read-only] int
TestRunner.Modifier inherit obj, implements TestRunner.Modifier System.IEquatable, System.Collections.IStructuralEquatable, TestRunner.Modifier System.IComparable, System.IComparable, System.Collections.IStructuralComparable - union type with 2 cases
TestRunner.Modifier+Explicit inherit TestRunner.Modifier
TestRunner.Modifier+Explicit.get_reason [method]: unit -> string option
TestRunner.Modifier+Explicit.reason [property]: [read-only] string option
TestRunner.Modifier+Ignored inherit TestRunner.Modifier
TestRunner.Modifier+Ignored.get_reason [method]: unit -> string option
TestRunner.Modifier+Ignored.reason [property]: [read-only] string option
TestRunner.Modifier+Tags inherit obj
TestRunner.Modifier+Tags.Explicit [static field]: int = 0
TestRunner.Modifier+Tags.Ignored [static field]: int = 1
TestRunner.Modifier.get_IsExplicit [method]: unit -> bool
TestRunner.Modifier.get_IsIgnored [method]: unit -> bool
TestRunner.Modifier.get_Tag [method]: unit -> int
TestRunner.Modifier.IsExplicit [property]: [read-only] bool
TestRunner.Modifier.IsIgnored [property]: [read-only] bool
TestRunner.Modifier.NewExplicit [static method]: string option -> TestRunner.Modifier
TestRunner.Modifier.NewIgnored [static method]: string option -> TestRunner.Modifier
TestRunner.Modifier.Tag [property]: [read-only] int
TestRunner.SingleTestMethod inherit obj, implements TestRunner.SingleTestMethod System.IEquatable, System.Collections.IStructuralEquatable
TestRunner.SingleTestMethod..ctor [constructor]: (System.Reflection.MethodInfo, TestRunner.TestKind, TestRunner.Modifier list, string list, int option, TestRunner.Combinatorial option)
TestRunner.SingleTestMethod.Categories [property]: [read-only] string list
TestRunner.SingleTestMethod.Combinatorial [property]: [read-only] TestRunner.Combinatorial option
TestRunner.SingleTestMethod.get_Categories [method]: unit -> string list
TestRunner.SingleTestMethod.get_Combinatorial [method]: unit -> TestRunner.Combinatorial option
TestRunner.SingleTestMethod.get_Kind [method]: unit -> TestRunner.TestKind
TestRunner.SingleTestMethod.get_Method [method]: unit -> System.Reflection.MethodInfo
TestRunner.SingleTestMethod.get_Modifiers [method]: unit -> TestRunner.Modifier list
TestRunner.SingleTestMethod.get_Name [method]: unit -> string
TestRunner.SingleTestMethod.get_Repeat [method]: unit -> int option
TestRunner.SingleTestMethod.Kind [property]: [read-only] TestRunner.TestKind
TestRunner.SingleTestMethod.Method [property]: [read-only] System.Reflection.MethodInfo
TestRunner.SingleTestMethod.Modifiers [property]: [read-only] TestRunner.Modifier list
TestRunner.SingleTestMethod.Name [property]: [read-only] string
TestRunner.SingleTestMethod.Repeat [property]: [read-only] int option
TestRunner.SingleTestMethodModule inherit obj
TestRunner.SingleTestMethodModule.parse [static method]: string list -> System.Reflection.MethodInfo -> System.Reflection.CustomAttributeData list -> (TestRunner.SingleTestMethod option * System.Reflection.CustomAttributeData list)
TestRunner.TestFailure inherit obj, implements TestRunner.TestFailure System.IEquatable, System.Collections.IStructuralEquatable - union type with 3 cases
TestRunner.TestFailure+SetUpFailed inherit TestRunner.TestFailure
TestRunner.TestFailure+SetUpFailed.get_Item [method]: unit -> TestRunner.UserMethodFailure
TestRunner.TestFailure+SetUpFailed.Item [property]: [read-only] TestRunner.UserMethodFailure
TestRunner.TestFailure+Tags inherit obj
TestRunner.TestFailure+Tags.SetUpFailed [static field]: int = 1
TestRunner.TestFailure+Tags.TearDownFailed [static field]: int = 2
TestRunner.TestFailure+Tags.TestFailed [static field]: int = 0
TestRunner.TestFailure+TearDownFailed inherit TestRunner.TestFailure
TestRunner.TestFailure+TearDownFailed.get_Item [method]: unit -> TestRunner.UserMethodFailure
TestRunner.TestFailure+TearDownFailed.Item [property]: [read-only] TestRunner.UserMethodFailure
TestRunner.TestFailure+TestFailed inherit TestRunner.TestFailure
TestRunner.TestFailure+TestFailed.get_Item [method]: unit -> TestRunner.UserMethodFailure
TestRunner.TestFailure+TestFailed.Item [property]: [read-only] TestRunner.UserMethodFailure
TestRunner.TestFailure.get_IsSetUpFailed [method]: unit -> bool
TestRunner.TestFailure.get_IsTearDownFailed [method]: unit -> bool
TestRunner.TestFailure.get_IsTestFailed [method]: unit -> bool
TestRunner.TestFailure.get_Name [method]: unit -> string
TestRunner.TestFailure.get_Tag [method]: unit -> int
TestRunner.TestFailure.IsSetUpFailed [property]: [read-only] bool
TestRunner.TestFailure.IsTearDownFailed [property]: [read-only] bool
TestRunner.TestFailure.IsTestFailed [property]: [read-only] bool
TestRunner.TestFailure.Name [property]: [read-only] string
TestRunner.TestFailure.NewSetUpFailed [static method]: TestRunner.UserMethodFailure -> TestRunner.TestFailure
TestRunner.TestFailure.NewTearDownFailed [static method]: TestRunner.UserMethodFailure -> TestRunner.TestFailure
TestRunner.TestFailure.NewTestFailed [static method]: TestRunner.UserMethodFailure -> TestRunner.TestFailure
TestRunner.TestFailure.Tag [property]: [read-only] int
TestRunner.TestFixture inherit obj, implements TestRunner.TestFixture System.IEquatable, System.Collections.IStructuralEquatable
TestRunner.TestFixture..ctor [constructor]: (string, System.Reflection.MethodInfo option, System.Reflection.MethodInfo option, System.Reflection.MethodInfo list, System.Reflection.MethodInfo list, TestRunner.SingleTestMethod list)
TestRunner.TestFixture.Empty [static method]: string -> TestRunner.TestFixture
TestRunner.TestFixture.get_Name [method]: unit -> string
TestRunner.TestFixture.get_OneTimeSetUp [method]: unit -> System.Reflection.MethodInfo option
TestRunner.TestFixture.get_OneTimeTearDown [method]: unit -> System.Reflection.MethodInfo option
TestRunner.TestFixture.get_SetUp [method]: unit -> System.Reflection.MethodInfo list
TestRunner.TestFixture.get_TearDown [method]: unit -> System.Reflection.MethodInfo list
TestRunner.TestFixture.get_Tests [method]: unit -> TestRunner.SingleTestMethod list
TestRunner.TestFixture.Name [property]: [read-only] string
TestRunner.TestFixture.OneTimeSetUp [property]: [read-only] System.Reflection.MethodInfo option
TestRunner.TestFixture.OneTimeTearDown [property]: [read-only] System.Reflection.MethodInfo option
TestRunner.TestFixture.SetUp [property]: [read-only] System.Reflection.MethodInfo list
TestRunner.TestFixture.TearDown [property]: [read-only] System.Reflection.MethodInfo list
TestRunner.TestFixture.Tests [property]: [read-only] TestRunner.SingleTestMethod list
TestRunner.TestFixtureModule inherit obj
TestRunner.TestFixtureModule.parse [static method]: System.Type -> TestRunner.TestFixture
TestRunner.TestFixtureModule.run [static method]: (TestRunner.TestFixture -> TestRunner.SingleTestMethod -> bool) -> TestRunner.TestFixture -> TestRunner.FixtureRunResults
TestRunner.TestKind inherit obj, implements TestRunner.TestKind System.IEquatable, System.Collections.IStructuralEquatable - union type with 3 cases
TestRunner.TestKind+Data inherit TestRunner.TestKind
TestRunner.TestKind+Data.get_Item [method]: unit -> obj list list
TestRunner.TestKind+Data.Item [property]: [read-only] obj list list
TestRunner.TestKind+Source inherit TestRunner.TestKind
TestRunner.TestKind+Source.get_Item [method]: unit -> string list
TestRunner.TestKind+Source.Item [property]: [read-only] string list
TestRunner.TestKind+Tags inherit obj
TestRunner.TestKind+Tags.Data [static field]: int = 2
TestRunner.TestKind+Tags.Single [static field]: int = 0
TestRunner.TestKind+Tags.Source [static field]: int = 1
TestRunner.TestKind.get_IsData [method]: unit -> bool
TestRunner.TestKind.get_IsSingle [method]: unit -> bool
TestRunner.TestKind.get_IsSource [method]: unit -> bool
TestRunner.TestKind.get_Single [static method]: unit -> TestRunner.TestKind
TestRunner.TestKind.get_Tag [method]: unit -> int
TestRunner.TestKind.IsData [property]: [read-only] bool
TestRunner.TestKind.IsSingle [property]: [read-only] bool
TestRunner.TestKind.IsSource [property]: [read-only] bool
TestRunner.TestKind.NewData [static method]: obj list list -> TestRunner.TestKind
TestRunner.TestKind.NewSource [static method]: string list -> TestRunner.TestKind
TestRunner.TestKind.Single [static property]: [read-only] TestRunner.TestKind
TestRunner.TestKind.Tag [property]: [read-only] int
TestRunner.TestMemberFailure inherit obj, implements TestRunner.TestMemberFailure System.IEquatable, System.Collections.IStructuralEquatable - union type with 2 cases
TestRunner.TestMemberFailure+Failed inherit TestRunner.TestMemberFailure
TestRunner.TestMemberFailure+Failed.get_Item [method]: unit -> TestRunner.TestFailure list
TestRunner.TestMemberFailure+Failed.Item [property]: [read-only] TestRunner.TestFailure list
TestRunner.TestMemberFailure+Malformed inherit TestRunner.TestMemberFailure
TestRunner.TestMemberFailure+Malformed.get_reasons [method]: unit -> string list
TestRunner.TestMemberFailure+Malformed.reasons [property]: [read-only] string list
TestRunner.TestMemberFailure+Tags inherit obj
TestRunner.TestMemberFailure+Tags.Failed [static field]: int = 1
TestRunner.TestMemberFailure+Tags.Malformed [static field]: int = 0
TestRunner.TestMemberFailure.get_IsFailed [method]: unit -> bool
TestRunner.TestMemberFailure.get_IsMalformed [method]: unit -> bool
TestRunner.TestMemberFailure.get_Tag [method]: unit -> int
TestRunner.TestMemberFailure.IsFailed [property]: [read-only] bool
TestRunner.TestMemberFailure.IsMalformed [property]: [read-only] bool
TestRunner.TestMemberFailure.NewFailed [static method]: TestRunner.TestFailure list -> TestRunner.TestMemberFailure
TestRunner.TestMemberFailure.NewMalformed [static method]: string list -> TestRunner.TestMemberFailure
TestRunner.TestMemberFailure.Tag [property]: [read-only] int
TestRunner.TestMemberSuccess inherit obj, implements TestRunner.TestMemberSuccess System.IEquatable, System.Collections.IStructuralEquatable, TestRunner.TestMemberSuccess System.IComparable, System.IComparable, System.Collections.IStructuralComparable - union type with 3 cases
TestRunner.TestMemberSuccess+Explicit inherit TestRunner.TestMemberSuccess
TestRunner.TestMemberSuccess+Explicit.get_reason [method]: unit -> string option
TestRunner.TestMemberSuccess+Explicit.reason [property]: [read-only] string option
TestRunner.TestMemberSuccess+Ignored inherit TestRunner.TestMemberSuccess
TestRunner.TestMemberSuccess+Ignored.get_reason [method]: unit -> string option
TestRunner.TestMemberSuccess+Ignored.reason [property]: [read-only] string option
TestRunner.TestMemberSuccess+Tags inherit obj
TestRunner.TestMemberSuccess+Tags.Explicit [static field]: int = 2
TestRunner.TestMemberSuccess+Tags.Ignored [static field]: int = 1
TestRunner.TestMemberSuccess+Tags.Ok [static field]: int = 0
TestRunner.TestMemberSuccess.get_IsExplicit [method]: unit -> bool
TestRunner.TestMemberSuccess.get_IsIgnored [method]: unit -> bool
TestRunner.TestMemberSuccess.get_IsOk [method]: unit -> bool
TestRunner.TestMemberSuccess.get_Ok [static method]: unit -> TestRunner.TestMemberSuccess
TestRunner.TestMemberSuccess.get_Tag [method]: unit -> int
TestRunner.TestMemberSuccess.IsExplicit [property]: [read-only] bool
TestRunner.TestMemberSuccess.IsIgnored [property]: [read-only] bool
TestRunner.TestMemberSuccess.IsOk [property]: [read-only] bool
TestRunner.TestMemberSuccess.NewExplicit [static method]: string option -> TestRunner.TestMemberSuccess
TestRunner.TestMemberSuccess.NewIgnored [static method]: string option -> TestRunner.TestMemberSuccess
TestRunner.TestMemberSuccess.Ok [static property]: [read-only] TestRunner.TestMemberSuccess
TestRunner.TestMemberSuccess.Tag [property]: [read-only] int
TestRunner.UserMethodFailure inherit obj, implements TestRunner.UserMethodFailure System.IEquatable, System.Collections.IStructuralEquatable - union type with 2 cases
TestRunner.UserMethodFailure+ReturnedNonUnit inherit TestRunner.UserMethodFailure
TestRunner.UserMethodFailure+ReturnedNonUnit.get_name [method]: unit -> string
TestRunner.UserMethodFailure+ReturnedNonUnit.get_result [method]: unit -> obj
TestRunner.UserMethodFailure+ReturnedNonUnit.name [property]: [read-only] string
TestRunner.UserMethodFailure+ReturnedNonUnit.result [property]: [read-only] obj
TestRunner.UserMethodFailure+Tags inherit obj
TestRunner.UserMethodFailure+Tags.ReturnedNonUnit [static field]: int = 0
TestRunner.UserMethodFailure+Tags.Threw [static field]: int = 1
TestRunner.UserMethodFailure+Threw inherit TestRunner.UserMethodFailure
TestRunner.UserMethodFailure+Threw.get_Item2 [method]: unit -> System.Exception
TestRunner.UserMethodFailure+Threw.get_name [method]: unit -> string
TestRunner.UserMethodFailure+Threw.Item2 [property]: [read-only] System.Exception
TestRunner.UserMethodFailure+Threw.name [property]: [read-only] string
TestRunner.UserMethodFailure.get_IsReturnedNonUnit [method]: unit -> bool
TestRunner.UserMethodFailure.get_IsThrew [method]: unit -> bool
TestRunner.UserMethodFailure.get_Name [method]: unit -> string
TestRunner.UserMethodFailure.get_Tag [method]: unit -> int
TestRunner.UserMethodFailure.IsReturnedNonUnit [property]: [read-only] bool
TestRunner.UserMethodFailure.IsThrew [property]: [read-only] bool
TestRunner.UserMethodFailure.Name [property]: [read-only] string
TestRunner.UserMethodFailure.NewReturnedNonUnit [static method]: (string, obj) -> TestRunner.UserMethodFailure
TestRunner.UserMethodFailure.NewThrew [static method]: (string, System.Exception) -> TestRunner.UserMethodFailure
TestRunner.UserMethodFailure.Tag [property]: [read-only] int

View File

@@ -5,17 +5,38 @@ open System.Reflection
open System.Threading
open Microsoft.FSharp.Core
/// Represents the result of a test that didn't fail.
[<RequireQualifiedAccess>]
type TestMemberSuccess =
/// The test passed.
| Ok
/// We didn't run the test, because it's [<Ignore>].
| Ignored of reason : string option
/// We didn't run the test, because it's [<Explicit>].
| Explicit of reason : string option
/// Represents the failure of a test.
[<RequireQualifiedAccess>]
type TestMemberFailure =
/// We couldn't run this test because it was somehow malformed in a way we detected up front.
| Malformed of reasons : string list
/// We tried to run the test, but it failed. (A single test can fail many times, e.g. if it failed and also
/// the tear-down logic failed afterwards.)
| Failed of TestFailure list
/// The results of running a single TestFixture.
type FixtureRunResults =
{
/// These tests failed.
Failed : TestMemberFailure list
/// This many tests succeeded (including multiple runs of a single test, if specified).
SuccessCount : int
/// These failures occurred outside the context of a test - e.g. in setup or tear-down logic.
OtherFailures : UserMethodFailure list
}
/// A test fixture (usually represented by the [<TestFixture>]` attribute), which may contain many tests,
/// each of which may run many times.
[<RequireQualifiedAccess>]
[<CompilationRepresentation(CompilationRepresentationFlags.ModuleSuffix)>]
module TestFixture =
@@ -42,8 +63,8 @@ module TestFixture =
let result =
try
head.Invoke (containingObject, args) |> Ok
with e ->
Error (UserMethodFailure.Threw (head.Name, e))
with :? TargetInvocationException as e ->
Error (UserMethodFailure.Threw (head.Name, e.InnerException))
match result with
| Error e -> Error (wrap e)
@@ -58,7 +79,8 @@ module TestFixture =
let result = runMethods TestFailure.TestFailed [ test ] args
let tearDownResult = runMethods TestFailure.TestFailed tearDown [||]
// Unconditionally run TearDown after tests, even if tests failed.
let tearDownResult = runMethods TestFailure.TearDownFailed tearDown [||]
match result, tearDownResult with
| Ok (), Ok () -> Ok ()
@@ -100,7 +122,8 @@ module TestFixture =
else
Ok None
/// This method only throws if there's a critical logic error in the runner.
/// This method should never throw: it only throws if there's a critical logic error in the runner.
/// Exceptions from the units under test are wrapped up and passed out.
let private runTestsFromMember
(setUp : MethodInfo list)
(tearDown : MethodInfo list)
@@ -244,29 +267,9 @@ module TestFixture =
|> Seq.concat
|> Seq.toList
let rec shouldRun (filter : Filter) : TestFixture -> SingleTestMethod -> bool =
match filter with
| Filter.Not filter ->
let inner = shouldRun filter
fun a b -> not (inner a b)
| Filter.And (a, b) ->
let inner1 = shouldRun a
let inner2 = shouldRun b
fun a b -> inner1 a b && inner2 a b
| Filter.Or (a, b) ->
let inner1 = shouldRun a
let inner2 = shouldRun b
fun a b -> inner1 a b || inner2 a b
| Filter.Name (Match.Exact m) -> fun _fixture method -> method.Method.Name = m
| Filter.Name (Match.Contains m) -> fun _fixture method -> method.Method.Name.Contains m
| Filter.FullyQualifiedName (Match.Exact m) -> fun fixture method -> (fixture.Name + method.Method.Name) = m
| Filter.FullyQualifiedName (Match.Contains m) ->
fun fixture method -> (fixture.Name + method.Method.Name).Contains m
| Filter.TestCategory (Match.Contains m) ->
fun _fixture method -> method.Categories |> List.exists (fun cat -> cat.Contains m)
| Filter.TestCategory (Match.Exact m) -> fun _fixture method -> method.Categories |> List.contains m
let run (filter : TestFixture -> SingleTestMethod -> bool) (tests : TestFixture) : int =
/// Run every test (except those which fail the `filter`) in this test fixture, as well as the
/// appropriate setup and tear-down logic.
let run (filter : TestFixture -> SingleTestMethod -> bool) (tests : TestFixture) : FixtureRunResults =
eprintfn $"Running test fixture: %s{tests.Name} (%i{tests.Tests.Length} tests to run)"
let containingObject =
@@ -292,17 +295,25 @@ module TestFixture =
)
|> Option.toObj
let setupResult =
match tests.OneTimeSetUp with
| Some su ->
try
match su.Invoke (containingObject, [||]) with
| :? unit -> ()
| ret -> failwith $"One-time setup procedure '%s{su.Name}' returned non-null %O{ret}"
| _ -> ()
| :? unit -> None
| ret -> Some (UserMethodFailure.ReturnedNonUnit (su.Name, ret))
with :? TargetInvocationException as e ->
Some (UserMethodFailure.Threw (su.Name, e.InnerException))
| _ -> None
let totalTestSuccess = ref 0
let testFailures = ref 0
let testFailures = ResizeArray ()
try
match setupResult with
| Some _ ->
// Don't run any tests if setup failed.
()
| None ->
for test in tests.Tests do
if filter tests test then
eprintfn $"Running test: %s{test.Name}"
@@ -312,29 +323,36 @@ module TestFixture =
for result in results do
match result with
| Error exc ->
eprintfn $"Test failed: %O{exc}"
Interlocked.Increment testFailures |> ignore<int>
| Error failure ->
testFailures.Add failure
eprintfn $"Test failed: %O{failure}"
| Ok _ -> Interlocked.Increment testSuccess |> ignore<int>
Interlocked.Add (totalTestSuccess, testSuccess.Value) |> ignore<int>
eprintfn $"Finished test %s{test.Name} (%i{testSuccess.Value} success)"
else
eprintfn $"Skipping test due to filter: %s{test.Name}"
finally
// Unconditionally run OneTimeTearDown if it exists.
let tearDownError =
match tests.OneTimeTearDown with
| Some td ->
try
// TODO: all these failwiths hide errors that we caught and wrapped up nicely above
if not (isNull (td.Invoke (containingObject, [||]))) then
failwith $"TearDown procedure '%s{td.Name}' returned non-null"
match td.Invoke (containingObject, [||]) with
| null -> None
| ret -> Some (UserMethodFailure.ReturnedNonUnit (td.Name, ret))
with :? TargetInvocationException as e ->
failwith $"One-time teardown of %s{td.Name} failed: %O{e.InnerException}"
| _ -> ()
Some (UserMethodFailure.Threw (td.Name, e))
| _ -> None
eprintfn $"Test fixture %s{tests.Name} completed (%i{totalTestSuccess.Value} success)."
testFailures.Value
{
Failed = testFailures |> Seq.toList
SuccessCount = totalTestSuccess.Value
OtherFailures = [ tearDownError ; setupResult ] |> List.choose id
}
/// Interpret this type as a [<TestFixture>], extracting the test members from it and annotating them with all
/// relevant information about how we should run them.
let parse (parentType : Type) : TestFixture =
let categories =
parentType.CustomAttributes

View File

@@ -17,16 +17,19 @@
</PropertyGroup>
<ItemGroup>
<Compile Include="AssemblyInfo.fs" />
<Compile Include="Array.fs" />
<Compile Include="Filter.fs" />
<Compile Include="List.fs" />
<Compile Include="Domain.fs" />
<Compile Include="Filter.fs" />
<Compile Include="SingleTestMethod.fs" />
<Compile Include="TestFixture.fs" />
<None Include="..\README.md">
<Pack>True</Pack>
<PackagePath>\</PackagePath>
</None>
<EmbeddedResource Include="SurfaceBaseline.txt" />
<EmbeddedResource Include="version.json" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="WoofWare.PrattParser" Version="0.1.2" />

View File

@@ -0,0 +1,9 @@
{
"version": "0.1",
"publicReleaseRefSpec": null,
"pathFilters": [
"./",
":/Directory.Build.props",
":/README.md"
]
}

View File

@@ -9,12 +9,12 @@ module Program =
let testDll, filter =
match argv |> List.ofSeq with
| [ dll ] -> FileInfo dll, None
| [ dll ; "--filter" ; filter ] -> FileInfo dll, Some (FilterIntermediate.parse filter |> Filter.make)
| [ dll ; "--filter" ; filter ] -> FileInfo dll, Some (Filter.parse filter)
| _ -> failwith "provide exactly one arg, a test DLL"
let filter =
match filter with
| Some filter -> TestFixture.shouldRun filter
| Some filter -> Filter.shouldRun filter
| None -> fun _ _ -> true
// Fix for https://github.com/Smaug123/unofficial-nunit-runner/issues/8
@@ -36,11 +36,27 @@ module Program =
(fun anyFailures ty ->
let testFixture = TestFixture.parse ty
match TestFixture.run filter testFixture with
| 0 -> anyFailures
| i ->
eprintfn $"%i{i} tests failed"
let results = TestFixture.run filter testFixture
let anyFailures =
match results.Failed with
| [] -> anyFailures
| _ :: _ ->
eprintfn $"%i{results.Failed.Length} tests failed"
true
let anyFailures =
match results.OtherFailures with
| [] -> anyFailures
| otherFailures ->
eprintfn "Other failures encountered: "
for failure in otherFailures do
eprintfn $" %s{failure.Name}"
true
anyFailures
)
false
finally

View File

@@ -10,68 +10,56 @@ module TestFilter =
let docExamples =
[
"(Name~MyClass) | (Name~MyClass2)",
FilterIntermediate.Or (
FilterIntermediate.Contains (FilterIntermediate.Name, FilterIntermediate.String "MyClass"),
FilterIntermediate.Contains (FilterIntermediate.Name, FilterIntermediate.String "MyClass2")
ParsedFilter.Or (
ParsedFilter.Contains (ParsedFilter.Name, ParsedFilter.String "MyClass"),
ParsedFilter.Contains (ParsedFilter.Name, ParsedFilter.String "MyClass2")
)
"xyz", FilterIntermediate.Contains (FilterIntermediate.FullyQualifiedName, FilterIntermediate.String "xyz")
"FullyQualifiedName~xyz",
FilterIntermediate.Contains (FilterIntermediate.FullyQualifiedName, FilterIntermediate.String "xyz")
"xyz", ParsedFilter.Contains (ParsedFilter.FullyQualifiedName, ParsedFilter.String "xyz")
"FullyQualifiedName~xyz", ParsedFilter.Contains (ParsedFilter.FullyQualifiedName, ParsedFilter.String "xyz")
"FullyQualifiedName!~IntegrationTests",
FilterIntermediate.Not (
FilterIntermediate.Contains (
FilterIntermediate.FullyQualifiedName,
FilterIntermediate.String "IntegrationTests"
)
ParsedFilter.Not (
ParsedFilter.Contains (ParsedFilter.FullyQualifiedName, ParsedFilter.String "IntegrationTests")
)
"FullyQualifiedName=MyNamespace.MyTestsClass<ParameterType1%2CParameterType2>.MyTestMethod",
FilterIntermediate.Equal (
FilterIntermediate.FullyQualifiedName,
FilterIntermediate.String "MyNamespace.MyTestsClass<ParameterType1%2CParameterType2>.MyTestMethod"
ParsedFilter.Equal (
ParsedFilter.FullyQualifiedName,
ParsedFilter.String "MyNamespace.MyTestsClass<ParameterType1%2CParameterType2>.MyTestMethod"
)
"Name~Method", FilterIntermediate.Contains (FilterIntermediate.Name, FilterIntermediate.String "Method")
"Name~Method", ParsedFilter.Contains (ParsedFilter.Name, ParsedFilter.String "Method")
"FullyQualifiedName!=MSTestNamespace.UnitTest1.TestMethod1",
FilterIntermediate.Not (
FilterIntermediate.Equal (
FilterIntermediate.FullyQualifiedName,
FilterIntermediate.String "MSTestNamespace.UnitTest1.TestMethod1"
ParsedFilter.Not (
ParsedFilter.Equal (
ParsedFilter.FullyQualifiedName,
ParsedFilter.String "MSTestNamespace.UnitTest1.TestMethod1"
)
)
"TestCategory=CategoryA",
FilterIntermediate.Equal (FilterIntermediate.TestCategory, FilterIntermediate.String "CategoryA")
"TestCategory=CategoryA", ParsedFilter.Equal (ParsedFilter.TestCategory, ParsedFilter.String "CategoryA")
"FullyQualifiedName~UnitTest1|TestCategory=CategoryA",
FilterIntermediate.Or (
FilterIntermediate.Contains (
FilterIntermediate.FullyQualifiedName,
FilterIntermediate.String "UnitTest1"
),
FilterIntermediate.Equal (FilterIntermediate.TestCategory, FilterIntermediate.String "CategoryA")
ParsedFilter.Or (
ParsedFilter.Contains (ParsedFilter.FullyQualifiedName, ParsedFilter.String "UnitTest1"),
ParsedFilter.Equal (ParsedFilter.TestCategory, ParsedFilter.String "CategoryA")
)
"FullyQualifiedName~UnitTest1&TestCategory=CategoryA",
FilterIntermediate.And (
FilterIntermediate.Contains (
FilterIntermediate.FullyQualifiedName,
FilterIntermediate.String "UnitTest1"
),
FilterIntermediate.Equal (FilterIntermediate.TestCategory, FilterIntermediate.String "CategoryA")
ParsedFilter.And (
ParsedFilter.Contains (ParsedFilter.FullyQualifiedName, ParsedFilter.String "UnitTest1"),
ParsedFilter.Equal (ParsedFilter.TestCategory, ParsedFilter.String "CategoryA")
)
"(FullyQualifiedName~UnitTest1&TestCategory=CategoryA)|TestCategory=1",
FilterIntermediate.Or (
FilterIntermediate.And (
FilterIntermediate.Contains (
FilterIntermediate.FullyQualifiedName,
FilterIntermediate.String "UnitTest1"
ParsedFilter.Or (
ParsedFilter.And (
ParsedFilter.Contains (ParsedFilter.FullyQualifiedName, ParsedFilter.String "UnitTest1"),
ParsedFilter.Equal (ParsedFilter.TestCategory, ParsedFilter.String "CategoryA")
),
FilterIntermediate.Equal (FilterIntermediate.TestCategory, FilterIntermediate.String "CategoryA")
),
FilterIntermediate.Equal (FilterIntermediate.TestCategory, FilterIntermediate.String "1")
ParsedFilter.Equal (ParsedFilter.TestCategory, ParsedFilter.String "1")
)
]
|> List.map TestCaseData
// sigh, NUnit doesn't want to run internal tests
[<TestCaseSource(nameof docExamples)>]
let ``Doc examples`` (example : string, expected : FilterIntermediate) =
FilterIntermediate.parse example |> shouldEqual expected
let ``Doc examples`` (example : string, expected : obj) =
let expected = expected |> unbox<ParsedFilter>
ParsedFilter.parse example |> shouldEqual expected
let docExamplesRefined =
[
@@ -112,4 +100,4 @@ module TestFilter =
[<TestCaseSource(nameof docExamplesRefined)>]
let ``Doc examples, refined`` (example : string, expected : Filter) =
FilterIntermediate.parse example |> Filter.make |> shouldEqual expected
Filter.parse example |> shouldEqual expected

View File

@@ -10,9 +10,11 @@
<ItemGroup>
<Compile Include="TestFilter.fs" />
<Compile Include="TestList.fs" />
<Compile Include="TestSurface.fs" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="ApiSurface" Version="4.0.40" />
<PackageReference Include="FsCheck" Version="3.0.0-rc3" />
<PackageReference Include="FsUnit" Version="6.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />

View File

@@ -0,0 +1,25 @@
namespace TestRunner.Test
open NUnit.Framework
open ApiSurface
[<TestFixture>]
module TestSurface =
let assembly = typeof<TestRunner.Combinatorial>.Assembly
[<Test>]
let ``Ensure API surface has not been modified`` () = ApiSurface.assertIdentical assembly
[<Test ; Explicit>]
let ``Update API surface`` () =
ApiSurface.writeAssemblyBaseline assembly
[<Test>]
let ``Ensure public API is fully documented`` () =
DocCoverage.assertFullyDocumented assembly
[<Test>]
[<Explicit "Not yet published">]
let ``Ensure version is monotonic`` () =
MonotonicVersion.validate assembly "CHOOSE A NAME"

10
TestRunner/version.json Normal file
View File

@@ -0,0 +1,10 @@
{
"version": "0.1",
"pathFilters": [
"./",
"^TestRunner.Test/",
"../TestRunner.Lib",
":/Directory.Build.props",
":/README.md"
]
}

View File

@@ -1,6 +1,11 @@
# This file was automatically generated by passthru.fetch-deps.
# Please dont edit it manually, your changes might get overwritten!
{fetchNuGet}: [
(fetchNuGet {
pname = "ApiSurface";
version = "4.0.40";
sha256 = "1c9z0b6minlripwrjmv4yd5w8zj4lcpak4x41izh7ygx8kgmbvx0";
})
(fetchNuGet {
pname = "fantomas";
version = "6.3.7";
@@ -16,6 +21,11 @@
version = "0.26.0";
sha256 = "0xgv5kvbwfdvcp6s8x7xagbbi4s3mqa4ixni6pazqvyflbgnah7b";
})
(fetchNuGet {
pname = "FSharp.Core";
version = "8.0.300";
sha256 = "158xxr9hnhz2ibyzzp2d249angvxfc58ifflm4g3hz8qx9zxaq04";
})
(fetchNuGet {
pname = "FsUnit";
version = "6.0.0";
@@ -31,6 +41,11 @@
version = "17.10.0";
sha256 = "13g8fwl09li8fc71nk13dgkb7gahd4qhamyg2xby7am63nlchhdf";
})
(fetchNuGet {
pname = "Microsoft.NETCore.Platforms";
version = "2.0.0";
sha256 = "1fk2fk2639i7nzy58m9dvpdnzql4vb8yl8vr19r2fp8lmj9w2jr0";
})
(fetchNuGet {
pname = "Microsoft.TestPlatform.ObjectModel";
version = "17.10.0";
@@ -41,11 +56,56 @@
version = "17.10.0";
sha256 = "1bl471s7fx9jycr0cc8rylwf34mrvlg9qn1an6l86nisavfcyb7v";
})
(fetchNuGet {
pname = "Nerdbank.GitVersioning";
version = "3.6.139";
sha256 = "0npcryhq3r0c2zi940jk39h13mzc4hyg7z8gm6jdmxi1aqv1vh8c";
})
(fetchNuGet {
pname = "NETStandard.Library.Ref";
version = "2.1.0";
sha256 = "12n76gymxq715lkrw841vi5r84kx746cxxssp22pd08as75jzsj6";
})
(fetchNuGet {
pname = "Newtonsoft.Json";
version = "13.0.1";
sha256 = "0fijg0w6iwap8gvzyjnndds0q4b8anwxxvik7y8vgq97dram4srb";
})
(fetchNuGet {
pname = "Newtonsoft.Json";
version = "13.0.3";
sha256 = "0xrwysmrn4midrjal8g2hr1bbg38iyisl0svamb11arqws4w2bw7";
})
(fetchNuGet {
pname = "NuGet.Common";
version = "6.10.0";
sha256 = "0nizrnilmlcqbm945293h8q3wfqfchb4xi8g50x4kjn0rbpd1kbh";
})
(fetchNuGet {
pname = "NuGet.Configuration";
version = "6.10.0";
sha256 = "1aqaknaawnqx4mnvx9qw73wvj48jjzv0d78dzwl7m9zjlrl9myhz";
})
(fetchNuGet {
pname = "NuGet.Frameworks";
version = "6.10.0";
sha256 = "0hrd8y31zx9a0wps49czw0qgbrakb49zn3abfgylc9xrq990zkqk";
})
(fetchNuGet {
pname = "NuGet.Packaging";
version = "6.10.0";
sha256 = "18s53cvrf51lihmaqqdf48p2qi6ky1l48jv0hvbp76cxwdg7rba4";
})
(fetchNuGet {
pname = "NuGet.Protocol";
version = "6.10.0";
sha256 = "0hmv4q0ks9i34mfgpb13l01la9v3jjllfh1qd3aqv105xrqrdxac";
})
(fetchNuGet {
pname = "NuGet.Versioning";
version = "6.10.0";
sha256 = "1x19njx4x0sw9fz8y5fibi15xfsrw5avir0cx0599yd7p3ykik5g";
})
(fetchNuGet {
pname = "NUnit";
version = "4.1.0";
@@ -56,11 +116,56 @@
version = "4.5.0";
sha256 = "1srx1629s0k1kmf02nmz251q07vj6pv58mdafcr5dr0bbn1fh78i";
})
(fetchNuGet {
pname = "System.Formats.Asn1";
version = "6.0.0";
sha256 = "1vvr7hs4qzjqb37r0w1mxq7xql2b17la63jwvmgv65s1hj00g8r9";
})
(fetchNuGet {
pname = "System.IO.Abstractions";
version = "4.2.13";
sha256 = "0s784iphsmj4vhkrzq9q3w39vsn76w44zclx3hsygsw458zbyh4y";
})
(fetchNuGet {
pname = "System.IO.FileSystem.AccessControl";
version = "4.5.0";
sha256 = "1gq4s8w7ds1sp8f9wqzf8nrzal40q5cd2w4pkf4fscrl2ih3hkkj";
})
(fetchNuGet {
pname = "System.Reflection.Metadata";
version = "1.6.0";
sha256 = "1wdbavrrkajy7qbdblpbpbalbdl48q3h34cchz24gvdgyrlf15r4";
})
(fetchNuGet {
pname = "System.Security.AccessControl";
version = "4.5.0";
sha256 = "1wvwanz33fzzbnd2jalar0p0z3x0ba53vzx1kazlskp7pwyhlnq0";
})
(fetchNuGet {
pname = "System.Security.Cryptography.Pkcs";
version = "6.0.4";
sha256 = "0hh5h38pnxmlrnvs72f2hzzpz4b2caiiv6xf8y7fzdg84r3imvfr";
})
(fetchNuGet {
pname = "System.Security.Cryptography.ProtectedData";
version = "4.4.0";
sha256 = "1q8ljvqhasyynp94a1d7jknk946m20lkwy2c3wa8zw2pc517fbj6";
})
(fetchNuGet {
pname = "System.Security.Principal.Windows";
version = "4.5.0";
sha256 = "0rmj89wsl5yzwh0kqjgx45vzf694v9p92r4x4q6yxldk1cv1hi86";
})
(fetchNuGet {
pname = "System.Text.Encodings.Web";
version = "7.0.0";
sha256 = "1151hbyrcf8kyg1jz8k9awpbic98lwz9x129rg7zk1wrs6vjlpxl";
})
(fetchNuGet {
pname = "System.Text.Json";
version = "7.0.3";
sha256 = "0zjrnc9lshagm6kdb9bdh45dmlnkpwcpyssa896sda93ngbmj8k9";
})
(fetchNuGet {
pname = "WoofWare.PrattParser";
version = "0.1.2";