Rejig test harness (#68)

This commit is contained in:
Patrick Stevens
2025-06-27 11:54:00 +01:00
committed by GitHub
parent 5cf0789439
commit c859f88f52
21 changed files with 467 additions and 50 deletions

View File

@@ -0,0 +1,18 @@
namespace WoofWare.PawPrint.Test
/// Result of executing the program using the real .NET runtime.
type RealRuntimeResult =
{
ExitCode : int
}
[<RequireQualifiedAccess>]
module RealRuntime =
/// Execute an assembly using the real .NET runtime and capture the result.
let executeWithRealRuntime (args : string[]) (assemblyBytes : byte array) : RealRuntimeResult =
let assy = System.Reflection.Assembly.Load assemblyBytes
let result = assy.EntryPoint.Invoke ((null : obj), [| args |]) |> unbox<int>
{
ExitCode = result
}

View File

@@ -32,7 +32,7 @@ module Roslyn =
|> Array.map (fun path -> MetadataReference.CreateFromFile path :> MetadataReference) |> Array.map (fun path -> MetadataReference.CreateFromFile path :> MetadataReference)
let compilationOptions = let compilationOptions =
CSharpCompilationOptions(OutputKind.ConsoleApplication).WithAllowUnsafe (true) CSharpCompilationOptions(OutputKind.ConsoleApplication).WithAllowUnsafe true
let compilation = let compilation =
CSharpCompilation.Create ( CSharpCompilation.Create (

View File

@@ -30,5 +30,5 @@ type TestCase =
FileName : string FileName : string
ExpectedReturnCode : int ExpectedReturnCode : int
NativeImpls : NativeImpls NativeImpls : NativeImpls
LocalVariablesOfMain : CliType list LocalVariablesOfMain : CliType list option
} }

View File

@@ -0,0 +1,123 @@
namespace WoofWare.Pawprint.Test
open System.Collections.Immutable
open System.IO
open FsUnitTyped
open NUnit.Framework
open WoofWare.DotnetRuntimeLocator
open WoofWare.PawPrint
open WoofWare.PawPrint.ExternImplementations
open WoofWare.PawPrint.Test
[<TestFixture>]
[<Parallelizable(ParallelScope.All)>]
module TestImpureCases =
let assy = typeof<RunResult>.Assembly
let unimplemented =
[
{
FileName = "WriteLine.cs"
ExpectedReturnCode = 1
NativeImpls = NativeImpls.PassThru ()
LocalVariablesOfMain = [] |> Some
}
]
let cases : TestCase list =
[
{
FileName = "InstaQuit.cs"
ExpectedReturnCode = 1
NativeImpls =
let mock = MockEnv.make ()
{ mock with
System_Environment =
{ System_EnvironmentMock.Empty with
GetProcessorCount =
fun thread state ->
let state =
state |> IlMachineState.pushToEvalStack' (EvalStackValue.Int32 1) thread
(state, WhatWeDid.Executed) |> ExecutionResult.Stepped
_Exit =
fun thread state ->
let state = state |> IlMachineState.loadArgument thread 0
ExecutionResult.Terminated (state, thread)
}
}
LocalVariablesOfMain = [] |> Some
}
]
[<TestCaseSource(nameof cases)>]
let ``Can evaluate C# files`` (case : TestCase) : unit =
let source = Assembly.getEmbeddedResourceAsString case.FileName assy
let image = Roslyn.compile [ source ]
let messages, loggerFactory = LoggerFactory.makeTest ()
let dotnetRuntimes =
DotnetRuntime.SelectForDll assy.Location |> ImmutableArray.CreateRange
use peImage = new MemoryStream (image)
try
let terminalState, terminatingThread =
Program.run loggerFactory (Some case.FileName) peImage dotnetRuntimes case.NativeImpls []
let exitCode =
match terminalState.ThreadState.[terminatingThread].MethodState.EvaluationStack.Values with
| [] -> failwith "expected program to return a value, but it returned void"
| head :: _ ->
match head with
| EvalStackValue.Int32 i -> i
| ret -> failwith $"expected program to return an int, but it returned %O{ret}"
exitCode |> shouldEqual case.ExpectedReturnCode
let finalVariables =
terminalState.ThreadState.[terminatingThread].MethodState.LocalVariables
|> Seq.toList
match case.LocalVariablesOfMain with
| None -> ()
| Some expected -> finalVariables |> shouldEqual expected
with _ ->
for message in messages () do
System.Console.Error.WriteLine $"{message}"
reraise ()
[<TestCaseSource(nameof unimplemented)>]
[<Explicit "not yet implemented">]
let ``Can evaluate C# files, unimplemented`` (case : TestCase) : unit =
let source = Assembly.getEmbeddedResourceAsString case.FileName assy
let image = Roslyn.compile [ source ]
let messages, loggerFactory = LoggerFactory.makeTest ()
let dotnetRuntimes =
DotnetRuntime.SelectForDll assy.Location |> ImmutableArray.CreateRange
use peImage = new MemoryStream (image)
try
let terminalState, terminatingThread =
Program.run loggerFactory (Some case.FileName) peImage dotnetRuntimes case.NativeImpls []
let exitCode =
match terminalState.ThreadState.[terminatingThread].MethodState.EvaluationStack.Values with
| [] -> failwith "expected program to return a value, but it returned void"
| head :: _ ->
match head with
| EvalStackValue.Int32 i -> i
| ret -> failwith $"expected program to return an int, but it returned %O{ret}"
exitCode |> shouldEqual case.ExpectedReturnCode
with _ ->
for message in messages () do
System.Console.Error.WriteLine $"{message}"
reraise ()

View File

@@ -0,0 +1,262 @@
namespace WoofWare.Pawprint.Test
open System.Collections.Immutable
open System.IO
open FsUnitTyped
open NUnit.Framework
open WoofWare.DotnetRuntimeLocator
open WoofWare.PawPrint
open WoofWare.PawPrint.ExternImplementations
open WoofWare.PawPrint.Test
[<TestFixture>]
[<Parallelizable(ParallelScope.All)>]
module TestPureCases =
let assy = typeof<RunResult>.Assembly
let unimplemented =
[
{
FileName = "Threads.cs"
ExpectedReturnCode = 3
NativeImpls = MockEnv.make ()
LocalVariablesOfMain = [] |> Some
}
{
FileName = "ComplexTryCatch.cs"
ExpectedReturnCode = 14
NativeImpls = NativeImpls.PassThru ()
LocalVariablesOfMain =
[
4
20
115
12
1
10
2
112
12
1111
42
99
25
50
123
20
35
5
11111
100001
]
|> List.map (fun i -> CliType.Numeric (CliNumericType.Int32 i))
|> Some
}
{
FileName = "ResizeArray.cs"
ExpectedReturnCode = 109
NativeImpls = MockEnv.make ()
LocalVariablesOfMain = [ CliType.Numeric (CliNumericType.Int32 10) ] |> Some
}
]
let cases : TestCase list =
[
{
FileName = "NoOp.cs"
ExpectedReturnCode = 1
NativeImpls = MockEnv.make ()
LocalVariablesOfMain = [ CliType.Numeric (CliNumericType.Int32 1) ] |> Some
}
{
FileName = "Ldind.cs"
ExpectedReturnCode = 0
NativeImpls = MockEnv.make ()
LocalVariablesOfMain =
[
// `failures`
CliType.Numeric (CliNumericType.Int32 0)
// Return value
CliType.Numeric (CliNumericType.Int32 0)
]
|> Some
}
{
FileName = "CustomDelegate.cs"
ExpectedReturnCode = 8
NativeImpls = MockEnv.make ()
LocalVariablesOfMain =
[
// filter
CliType.ObjectRef (Some (ManagedHeapAddress 2))
// result
CliType.OfBool true
// result, cloned for "if(result)" check
CliType.OfBool true
// ret
CliType.Numeric (CliNumericType.Int32 8)
]
|> Some
}
{
FileName = "ArgumentOrdering.cs"
ExpectedReturnCode = 42
NativeImpls = MockEnv.make ()
LocalVariablesOfMain =
[
// localVar
CliType.Numeric (CliNumericType.Int32 42)
// t
CliType.ValueType [ CliType.Numeric (CliNumericType.Int32 42) ]
// return value
CliType.Numeric (CliNumericType.Int32 42)
]
|> Some
}
{
FileName = "BasicLock.cs"
ExpectedReturnCode = 1
NativeImpls =
let mock = MockEnv.make ()
{ mock with
System_Threading_Monitor = System_Threading_Monitor.passThru
}
LocalVariablesOfMain =
[
// Four variables:
// locker
CliType.ObjectRef (Some (ManagedHeapAddress 2))
// a copy of locker, taken so that the contents of the implicit `finally` have a stable copy
CliType.ObjectRef (Some (ManagedHeapAddress 2))
// out param of `ReliableEnter`
CliType.OfBool true
// return value
CliType.Numeric (CliNumericType.Int32 1)
]
|> Some
}
{
FileName = "TriangleNumber.cs"
ExpectedReturnCode = 10
NativeImpls = MockEnv.make ()
LocalVariablesOfMain =
[
// answer
CliType.Numeric (CliNumericType.Int32 10)
// i
CliType.Numeric (CliNumericType.Int32 5)
// End-loop condition
CliType.OfBool false
// Ret
CliType.Numeric (CliNumericType.Int32 10)
]
|> Some
}
{
FileName = "ExceptionWithNoOpFinally.cs"
ExpectedReturnCode = 3
NativeImpls = MockEnv.make ()
LocalVariablesOfMain =
[
// Variable 1 is `x`, variable 2 is the implicit return value
4
3
]
|> List.map (fun i -> CliType.Numeric (CliNumericType.Int32 i))
|> Some
}
{
FileName = "ExceptionWithNoOpCatch.cs"
ExpectedReturnCode = 10
NativeImpls = MockEnv.make ()
LocalVariablesOfMain = [ CliType.Numeric (CliNumericType.Int32 10) ] |> Some
}
{
FileName = "TryCatchWithThrowInBody.cs"
ExpectedReturnCode = 4
NativeImpls = MockEnv.make ()
LocalVariablesOfMain =
[
// one variable is x, one variable is the return value which also happens to have the same value
4
4
]
|> List.map (fun i -> CliType.Numeric (CliNumericType.Int32 i))
|> Some
}
]
[<TestCaseSource(nameof cases)>]
let ``Can evaluate C# files`` (case : TestCase) : unit =
let source = Assembly.getEmbeddedResourceAsString case.FileName assy
let image = Roslyn.compile [ source ]
let messages, loggerFactory = LoggerFactory.makeTest ()
let dotnetRuntimes =
DotnetRuntime.SelectForDll assy.Location |> ImmutableArray.CreateRange
use peImage = new MemoryStream (image)
try
let terminalState, terminatingThread =
Program.run loggerFactory (Some case.FileName) peImage dotnetRuntimes case.NativeImpls []
let realResult = RealRuntime.executeWithRealRuntime [||] image
let exitCode =
match terminalState.ThreadState.[terminatingThread].MethodState.EvaluationStack.Values with
| [] -> failwith "expected program to return a value, but it returned void"
| head :: _ ->
match head with
| EvalStackValue.Int32 i -> i
| ret -> failwith $"expected program to return an int, but it returned %O{ret}"
realResult.ExitCode |> shouldEqual exitCode
exitCode |> shouldEqual case.ExpectedReturnCode
let finalVariables =
terminalState.ThreadState.[terminatingThread].MethodState.LocalVariables
|> Seq.toList
match case.LocalVariablesOfMain with
| None -> ()
| Some expected -> finalVariables |> shouldEqual expected
with _ ->
for message in messages () do
System.Console.Error.WriteLine $"{message}"
reraise ()
[<TestCaseSource(nameof unimplemented)>]
[<Explicit "not yet implemented">]
let ``Can evaluate C# files, unimplemented`` (case : TestCase) : unit =
let source = Assembly.getEmbeddedResourceAsString case.FileName assy
let image = Roslyn.compile [ source ]
let messages, loggerFactory = LoggerFactory.makeTest ()
let dotnetRuntimes =
DotnetRuntime.SelectForDll assy.Location |> ImmutableArray.CreateRange
use peImage = new MemoryStream (image)
try
let terminalState, terminatingThread =
Program.run loggerFactory (Some case.FileName) peImage dotnetRuntimes case.NativeImpls []
let exitCode =
match terminalState.ThreadState.[terminatingThread].MethodState.EvaluationStack.Values with
| [] -> failwith "expected program to return a value, but it returned void"
| head :: _ ->
match head with
| EvalStackValue.Int32 i -> i
| ret -> failwith $"expected program to return an int, but it returned %O{ret}"
exitCode |> shouldEqual case.ExpectedReturnCode
with _ ->
for message in messages () do
System.Console.Error.WriteLine $"{message}"
reraise ()

View File

@@ -13,23 +13,28 @@
<Compile Include="LoggerFactory.fs" /> <Compile Include="LoggerFactory.fs" />
<Compile Include="Assembly.fs" /> <Compile Include="Assembly.fs" />
<Compile Include="Roslyn.fs" /> <Compile Include="Roslyn.fs" />
<Compile Include="RealRuntime.fs" />
<Compile Include="TestHarness.fs"/> <Compile Include="TestHarness.fs"/>
<Compile Include="TestCases.fs" /> <Compile Include="TestPureCases.fs" />
<Compile Include="TestHelloWorld.fs" /> <Compile Include="TestImpureCases.fs" />
<EmbeddedResource Include="sources\BasicLock.cs" /> </ItemGroup>
<EmbeddedResource Include="sources\NoOp.cs" /> <ItemGroup>
<EmbeddedResource Include="sources\ExceptionWithNoOpCatch.cs" /> <EmbeddedResource Include="sourcesPure\BasicLock.cs" />
<EmbeddedResource Include="sources\ExceptionWithNoOpFinally.cs" /> <EmbeddedResource Include="sourcesPure\NoOp.cs" />
<EmbeddedResource Include="sources\TryCatchWithThrowInBody.cs" /> <EmbeddedResource Include="sourcesPure\ExceptionWithNoOpCatch.cs" />
<EmbeddedResource Include="sources\ComplexTryCatch.cs" /> <EmbeddedResource Include="sourcesPure\ExceptionWithNoOpFinally.cs" />
<EmbeddedResource Include="sources\TriangleNumber.cs" /> <EmbeddedResource Include="sourcesPure\TryCatchWithThrowInBody.cs" />
<EmbeddedResource Include="sources\WriteLine.cs" /> <EmbeddedResource Include="sourcesPure\ComplexTryCatch.cs" />
<EmbeddedResource Include="sources\InstaQuit.cs" /> <EmbeddedResource Include="sourcesPure\TriangleNumber.cs" />
<EmbeddedResource Include="sources\Threads.cs" /> <EmbeddedResource Include="sourcesPure\Threads.cs" />
<EmbeddedResource Include="sources\ResizeArray.cs" /> <EmbeddedResource Include="sourcesPure\ResizeArray.cs" />
<EmbeddedResource Include="sources\ArgumentOrdering.cs" /> <EmbeddedResource Include="sourcesPure\ArgumentOrdering.cs" />
<EmbeddedResource Include="sources\CustomDelegate.cs" /> <EmbeddedResource Include="sourcesPure\CustomDelegate.cs" />
<EmbeddedResource Include="sources\Ldind.cs" /> <EmbeddedResource Include="sourcesPure\Ldind.cs" />
</ItemGroup>
<ItemGroup>
<EmbeddedResource Include="sourcesImpure\WriteLine.cs" />
<EmbeddedResource Include="sourcesImpure\InstaQuit.cs" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
@@ -37,14 +42,13 @@
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="FsUnit" Version="7.0.1"/> <PackageReference Include="FsUnit" Version="7.1.1"/>
<PackageReference Include="NUnit3TestAdapter" Version="5.0.0"/> <PackageReference Include="NUnit3TestAdapter" Version="5.0.0"/>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.13.0"/> <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1"/>
<PackageReference Include="NUnit" Version="4.3.2"/> <PackageReference Include="NUnit" Version="4.3.2"/>
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.8.0"/> <PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.14.0"/>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.2" /> <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.6" />
<PackageReference Include="WoofWare.DotnetRuntimeLocator" Version="0.3.2"/> <PackageReference Include="WoofWare.DotnetRuntimeLocator" Version="0.3.2"/>
</ItemGroup> </ItemGroup>
</Project> </Project>

View File

@@ -26,8 +26,8 @@
}, },
{ {
"pname": "FsUnit", "pname": "FsUnit",
"version": "7.0.1", "version": "7.1.1",
"hash": "sha256-K85CIdxMeFSHEKZk6heIXp/oFjWAn7dBILKrw49pJUY=" "hash": "sha256-UMCEGKxQ4ytjmPuVpiNaAPbi3RQH9gqa61JJIUS/6hg="
}, },
{ {
"pname": "Microsoft.ApplicationInsights", "pname": "Microsoft.ApplicationInsights",
@@ -61,38 +61,48 @@
}, },
{ {
"pname": "Microsoft.CodeAnalysis.Analyzers", "pname": "Microsoft.CodeAnalysis.Analyzers",
"version": "3.3.4", "version": "3.11.0",
"hash": "sha256-qDzTfZBSCvAUu9gzq2k+LOvh6/eRvJ9++VCNck/ZpnE=" "hash": "sha256-hQ2l6E6PO4m7i+ZsfFlEx+93UsLPo4IY3wDkNG11/Sw="
}, },
{ {
"pname": "Microsoft.CodeAnalysis.Common", "pname": "Microsoft.CodeAnalysis.Common",
"version": "4.8.0", "version": "4.14.0",
"hash": "sha256-3IEinVTZq6/aajMVA8XTRO3LTIEt0PuhGyITGJLtqz4=" "hash": "sha256-ne/zxH3GqoGB4OemnE8oJElG5mai+/67ASaKqwmL2BE="
}, },
{ {
"pname": "Microsoft.CodeAnalysis.CSharp", "pname": "Microsoft.CodeAnalysis.CSharp",
"version": "4.8.0", "version": "4.14.0",
"hash": "sha256-MmOnXJvd/ezs5UPcqyGLnbZz5m+VedpRfB+kFZeeqkU=" "hash": "sha256-5Mzj3XkYYLkwDWh17r1NEXSbXwwWYQPiOmkSMlgo1JY="
}, },
{ {
"pname": "Microsoft.CodeCoverage", "pname": "Microsoft.CodeCoverage",
"version": "17.13.0", "version": "17.14.1",
"hash": "sha256-GKrIxeyQo5Az1mztfQgea1kGtJwonnNOrXK/0ULfu8o=" "hash": "sha256-f8QytG8GvRoP47rO2KEmnDLxIpyesaq26TFjDdW40Gs="
}, },
{ {
"pname": "Microsoft.Extensions.DependencyInjection.Abstractions", "pname": "Microsoft.Extensions.DependencyInjection.Abstractions",
"version": "9.0.2", "version": "9.0.2",
"hash": "sha256-WoTLgw/OlXhgN54Szip0Zpne7i/YTXwZ1ZLCPcHV6QM=" "hash": "sha256-WoTLgw/OlXhgN54Szip0Zpne7i/YTXwZ1ZLCPcHV6QM="
}, },
{
"pname": "Microsoft.Extensions.DependencyInjection.Abstractions",
"version": "9.0.6",
"hash": "sha256-40rY38OwSqueIWr/KMvJX9u+vipN+AaRQ6eNCZLqrog="
},
{ {
"pname": "Microsoft.Extensions.Logging.Abstractions", "pname": "Microsoft.Extensions.Logging.Abstractions",
"version": "9.0.2", "version": "9.0.2",
"hash": "sha256-mCxeuc+37XY0bmZR+z4p1hrZUdTZEg+FRcs/m6dAQDU=" "hash": "sha256-mCxeuc+37XY0bmZR+z4p1hrZUdTZEg+FRcs/m6dAQDU="
}, },
{
"pname": "Microsoft.Extensions.Logging.Abstractions",
"version": "9.0.6",
"hash": "sha256-lhOMYT4+hua7SlgASGFBDhOkrNOsy35WyIxU3nVsD08="
},
{ {
"pname": "Microsoft.NET.Test.Sdk", "pname": "Microsoft.NET.Test.Sdk",
"version": "17.13.0", "version": "17.14.1",
"hash": "sha256-sc2wvyV8cGm1FrNP2GGHEI584RCvRPu15erYCsgw5QY=" "hash": "sha256-mZUzDFvFp7x1nKrcnRd0hhbNu5g8EQYt8SKnRgdhT/A="
}, },
{ {
"pname": "Microsoft.NETCore.App.Host.linux-arm64", "pname": "Microsoft.NETCore.App.Host.linux-arm64",
@@ -191,13 +201,13 @@
}, },
{ {
"pname": "Microsoft.TestPlatform.ObjectModel", "pname": "Microsoft.TestPlatform.ObjectModel",
"version": "17.13.0", "version": "17.14.1",
"hash": "sha256-6S0fjfj8vA+h6dJVNwLi6oZhYDO/I/6hBZaq2VTW+Uk=" "hash": "sha256-QMf6O+w0IT+16Mrzo7wn+N20f3L1/mDhs/qjmEo1rYs="
}, },
{ {
"pname": "Microsoft.TestPlatform.TestHost", "pname": "Microsoft.TestPlatform.TestHost",
"version": "17.13.0", "version": "17.14.1",
"hash": "sha256-L/CJzou7dhmShUgXq3aXL3CaLTJll17Q+JY2DBdUUpo=" "hash": "sha256-1cxHWcvHRD7orQ3EEEPPxVGEkTpxom1/zoICC9SInJs="
}, },
{ {
"pname": "Myriad.Core", "pname": "Myriad.Core",
@@ -216,8 +226,8 @@
}, },
{ {
"pname": "Newtonsoft.Json", "pname": "Newtonsoft.Json",
"version": "13.0.1", "version": "13.0.3",
"hash": "sha256-K2tSVW4n4beRPzPu3rlVaBEMdGvWSv/3Q1fxaDh4Mjo=" "hash": "sha256-hy/BieY4qxBWVVsDqqOPaLy1QobiIapkbrESm6v2PHc="
}, },
{ {
"pname": "NUnit", "pname": "NUnit",
@@ -246,8 +256,8 @@
}, },
{ {
"pname": "System.Collections.Immutable", "pname": "System.Collections.Immutable",
"version": "7.0.0", "version": "9.0.0",
"hash": "sha256-9an2wbxue2qrtugYES9awshQg+KfJqajhnhs45kQIdk=" "hash": "sha256-+6q5VMeoc5bm4WFsoV6nBXA9dV5pa/O4yW+gOdi8yac="
}, },
{ {
"pname": "System.Diagnostics.DiagnosticSource", "pname": "System.Diagnostics.DiagnosticSource",
@@ -281,19 +291,19 @@
}, },
{ {
"pname": "System.Reflection.Metadata", "pname": "System.Reflection.Metadata",
"version": "7.0.0", "version": "8.0.0",
"hash": "sha256-GwAKQhkhPBYTqmRdG9c9taqrKSKDwyUgOEhWLKxWNPI=" "hash": "sha256-dQGC30JauIDWNWXMrSNOJncVa1umR1sijazYwUDdSIE="
},
{
"pname": "System.Reflection.Metadata",
"version": "9.0.0",
"hash": "sha256-avEWbcCh7XgpsSesnR3/SgxWi/6C5OxjR89Jf/SfRjQ="
}, },
{ {
"pname": "System.Runtime", "pname": "System.Runtime",
"version": "4.3.1", "version": "4.3.1",
"hash": "sha256-R9T68AzS1PJJ7v6ARz9vo88pKL1dWqLOANg4pkQjkA0=" "hash": "sha256-R9T68AzS1PJJ7v6ARz9vo88pKL1dWqLOANg4pkQjkA0="
}, },
{
"pname": "System.Runtime.CompilerServices.Unsafe",
"version": "6.0.0",
"hash": "sha256-bEG1PnDp7uKYz/OgLOWs3RWwQSVYm+AnPwVmAmcgp2I="
},
{ {
"pname": "TypeEquality", "pname": "TypeEquality",
"version": "0.3.0", "version": "0.3.0",