Rejig test harness (#68)

WoofWare.PawPrint.Test/RealRuntime.fs

@@ -0,0 +1,18 @@
+namespace WoofWare.PawPrint.Test
+
+/// Result of executing the program using the real .NET runtime.
+type RealRuntimeResult =
+    {
+        ExitCode : int
+    }
+
+[<RequireQualifiedAccess>]
+module RealRuntime =
+    /// Execute an assembly using the real .NET runtime and capture the result.
+    let executeWithRealRuntime (args : string[]) (assemblyBytes : byte array) : RealRuntimeResult =
+        let assy = System.Reflection.Assembly.Load assemblyBytes
+        let result = assy.EntryPoint.Invoke ((null : obj), [| args |]) |> unbox<int>
+
+        {
+            ExitCode = result
+        }

WoofWare.PawPrint.Test/Roslyn.fs

@@ -32,7 +32,7 @@ module Roslyn =
             |> Array.map (fun path -> MetadataReference.CreateFromFile path :> MetadataReference)
 
         let compilationOptions =
-            CSharpCompilationOptions(OutputKind.ConsoleApplication).WithAllowUnsafe (true)
+            CSharpCompilationOptions(OutputKind.ConsoleApplication).WithAllowUnsafe true
 
         let compilation =
             CSharpCompilation.Create (

WoofWare.PawPrint.Test/TestHarness.fs

@@ -30,5 +30,5 @@ type TestCase =
         FileName : string
         ExpectedReturnCode : int
         NativeImpls : NativeImpls
-        LocalVariablesOfMain : CliType list
+        LocalVariablesOfMain : CliType list option
     }

WoofWare.PawPrint.Test/TestImpureCases.fs

@@ -0,0 +1,123 @@
+namespace WoofWare.Pawprint.Test
+
+open System.Collections.Immutable
+open System.IO
+open FsUnitTyped
+open NUnit.Framework
+open WoofWare.DotnetRuntimeLocator
+open WoofWare.PawPrint
+open WoofWare.PawPrint.ExternImplementations
+open WoofWare.PawPrint.Test
+
+[<TestFixture>]
+[<Parallelizable(ParallelScope.All)>]
+module TestImpureCases =
+    let assy = typeof<RunResult>.Assembly
+
+    let unimplemented =
+        [
+            {
+                FileName = "WriteLine.cs"
+                ExpectedReturnCode = 1
+                NativeImpls = NativeImpls.PassThru ()
+                LocalVariablesOfMain = [] |> Some
+            }
+        ]
+
+    let cases : TestCase list =
+        [
+            {
+                FileName = "InstaQuit.cs"
+                ExpectedReturnCode = 1
+                NativeImpls =
+                    let mock = MockEnv.make ()
+
+                    { mock with
+                        System_Environment =
+                            { System_EnvironmentMock.Empty with
+                                GetProcessorCount =
+                                    fun thread state ->
+                                        let state =
+                                            state |> IlMachineState.pushToEvalStack' (EvalStackValue.Int32 1) thread
+
+                                        (state, WhatWeDid.Executed) |> ExecutionResult.Stepped
+                                _Exit =
+                                    fun thread state ->
+                                        let state = state |> IlMachineState.loadArgument thread 0
+                                        ExecutionResult.Terminated (state, thread)
+                            }
+                    }
+                LocalVariablesOfMain = [] |> Some
+            }
+        ]
+
+    [<TestCaseSource(nameof cases)>]
+    let ``Can evaluate C# files`` (case : TestCase) : unit =
+        let source = Assembly.getEmbeddedResourceAsString case.FileName assy
+        let image = Roslyn.compile [ source ]
+        let messages, loggerFactory = LoggerFactory.makeTest ()
+
+        let dotnetRuntimes =
+            DotnetRuntime.SelectForDll assy.Location |> ImmutableArray.CreateRange
+
+        use peImage = new MemoryStream (image)
+
+        try
+            let terminalState, terminatingThread =
+                Program.run loggerFactory (Some case.FileName) peImage dotnetRuntimes case.NativeImpls []
+
+            let exitCode =
+                match terminalState.ThreadState.[terminatingThread].MethodState.EvaluationStack.Values with
+                | [] -> failwith "expected program to return a value, but it returned void"
+                | head :: _ ->
+                    match head with
+                    | EvalStackValue.Int32 i -> i
+                    | ret -> failwith $"expected program to return an int, but it returned %O{ret}"
+
+            exitCode |> shouldEqual case.ExpectedReturnCode
+
+            let finalVariables =
+                terminalState.ThreadState.[terminatingThread].MethodState.LocalVariables
+                |> Seq.toList
+
+            match case.LocalVariablesOfMain with
+            | None -> ()
+            | Some expected -> finalVariables |> shouldEqual expected
+
+        with _ ->
+            for message in messages () do
+                System.Console.Error.WriteLine $"{message}"
+
+            reraise ()
+
+    [<TestCaseSource(nameof unimplemented)>]
+    [<Explicit "not yet implemented">]
+    let ``Can evaluate C# files, unimplemented`` (case : TestCase) : unit =
+        let source = Assembly.getEmbeddedResourceAsString case.FileName assy
+        let image = Roslyn.compile [ source ]
+        let messages, loggerFactory = LoggerFactory.makeTest ()
+
+        let dotnetRuntimes =
+            DotnetRuntime.SelectForDll assy.Location |> ImmutableArray.CreateRange
+
+        use peImage = new MemoryStream (image)
+
+        try
+            let terminalState, terminatingThread =
+                Program.run loggerFactory (Some case.FileName) peImage dotnetRuntimes case.NativeImpls []
+
+            let exitCode =
+                match terminalState.ThreadState.[terminatingThread].MethodState.EvaluationStack.Values with
+                | [] -> failwith "expected program to return a value, but it returned void"
+                | head :: _ ->
+                    match head with
+                    | EvalStackValue.Int32 i -> i
+                    | ret -> failwith $"expected program to return an int, but it returned %O{ret}"
+
+            exitCode |> shouldEqual case.ExpectedReturnCode
+
+        with _ ->
+            for message in messages () do
+                System.Console.Error.WriteLine $"{message}"
+
+            reraise ()

WoofWare.PawPrint.Test/TestPureCases.fs

@@ -0,0 +1,262 @@
+namespace WoofWare.Pawprint.Test
+
+open System.Collections.Immutable
+open System.IO
+open FsUnitTyped
+open NUnit.Framework
+open WoofWare.DotnetRuntimeLocator
+open WoofWare.PawPrint
+open WoofWare.PawPrint.ExternImplementations
+open WoofWare.PawPrint.Test
+
+[<TestFixture>]
+[<Parallelizable(ParallelScope.All)>]
+module TestPureCases =
+    let assy = typeof<RunResult>.Assembly
+
+    let unimplemented =
+        [
+            {
+                FileName = "Threads.cs"
+                ExpectedReturnCode = 3
+                NativeImpls = MockEnv.make ()
+                LocalVariablesOfMain = [] |> Some
+            }
+            {
+                FileName = "ComplexTryCatch.cs"
+                ExpectedReturnCode = 14
+                NativeImpls = NativeImpls.PassThru ()
+                LocalVariablesOfMain =
+                    [
+                        4
+                        20
+                        115
+                        12
+                        1
+                        10
+                        2
+                        112
+                        12
+                        1111
+                        42
+                        99
+                        25
+                        50
+                        123
+                        20
+                        35
+                        5
+                        11111
+                        100001
+                    ]
+                    |> List.map (fun i -> CliType.Numeric (CliNumericType.Int32 i))
+                    |> Some
+            }
+            {
+                FileName = "ResizeArray.cs"
+                ExpectedReturnCode = 109
+                NativeImpls = MockEnv.make ()
+                LocalVariablesOfMain = [ CliType.Numeric (CliNumericType.Int32 10) ] |> Some
+            }
+        ]
+
+    let cases : TestCase list =
+        [
+            {
+                FileName = "NoOp.cs"
+                ExpectedReturnCode = 1
+                NativeImpls = MockEnv.make ()
+                LocalVariablesOfMain = [ CliType.Numeric (CliNumericType.Int32 1) ] |> Some
+            }
+            {
+                FileName = "Ldind.cs"
+                ExpectedReturnCode = 0
+                NativeImpls = MockEnv.make ()
+                LocalVariablesOfMain =
+                    [
+                        // `failures`
+                        CliType.Numeric (CliNumericType.Int32 0)
+                        // Return value
+                        CliType.Numeric (CliNumericType.Int32 0)
+                    ]
+                    |> Some
+            }
+            {
+                FileName = "CustomDelegate.cs"
+                ExpectedReturnCode = 8
+                NativeImpls = MockEnv.make ()
+                LocalVariablesOfMain =
+                    [
+                        // filter
+                        CliType.ObjectRef (Some (ManagedHeapAddress 2))
+                        // result
+                        CliType.OfBool true
+                        // result, cloned for "if(result)" check
+                        CliType.OfBool true
+                        // ret
+                        CliType.Numeric (CliNumericType.Int32 8)
+                    ]
+                    |> Some
+            }
+            {
+                FileName = "ArgumentOrdering.cs"
+                ExpectedReturnCode = 42
+                NativeImpls = MockEnv.make ()
+                LocalVariablesOfMain =
+                    [
+                        // localVar
+                        CliType.Numeric (CliNumericType.Int32 42)
+                        // t
+                        CliType.ValueType [ CliType.Numeric (CliNumericType.Int32 42) ]
+                        // return value
+                        CliType.Numeric (CliNumericType.Int32 42)
+                    ]
+                    |> Some
+            }
+            {
+                FileName = "BasicLock.cs"
+                ExpectedReturnCode = 1
+                NativeImpls =
+                    let mock = MockEnv.make ()
+
+                    { mock with
+                        System_Threading_Monitor = System_Threading_Monitor.passThru
+                    }
+                LocalVariablesOfMain =
+                    [
+                        // Four variables:
+                        // locker
+                        CliType.ObjectRef (Some (ManagedHeapAddress 2))
+                        // a copy of locker, taken so that the contents of the implicit `finally` have a stable copy
+                        CliType.ObjectRef (Some (ManagedHeapAddress 2))
+                        // out param of `ReliableEnter`
+                        CliType.OfBool true
+                        // return value
+                        CliType.Numeric (CliNumericType.Int32 1)
+                    ]
+                    |> Some
+            }
+            {
+                FileName = "TriangleNumber.cs"
+                ExpectedReturnCode = 10
+                NativeImpls = MockEnv.make ()
+                LocalVariablesOfMain =
+                    [
+                        // answer
+                        CliType.Numeric (CliNumericType.Int32 10)
+                        // i
+                        CliType.Numeric (CliNumericType.Int32 5)
+                        // End-loop condition
+                        CliType.OfBool false
+                        // Ret
+                        CliType.Numeric (CliNumericType.Int32 10)
+                    ]
+                    |> Some
+            }
+            {
+                FileName = "ExceptionWithNoOpFinally.cs"
+                ExpectedReturnCode = 3
+                NativeImpls = MockEnv.make ()
+                LocalVariablesOfMain =
+                    [
+                        // Variable 1 is `x`, variable 2 is the implicit return value
+                        4
+                        3
+                    ]
+                    |> List.map (fun i -> CliType.Numeric (CliNumericType.Int32 i))
+                    |> Some
+            }
+            {
+                FileName = "ExceptionWithNoOpCatch.cs"
+                ExpectedReturnCode = 10
+                NativeImpls = MockEnv.make ()
+                LocalVariablesOfMain = [ CliType.Numeric (CliNumericType.Int32 10) ] |> Some
+            }
+            {
+                FileName = "TryCatchWithThrowInBody.cs"
+                ExpectedReturnCode = 4
+                NativeImpls = MockEnv.make ()
+                LocalVariablesOfMain =
+                    [
+                        // one variable is x, one variable is the return value which also happens to have the same value
+                        4
+                        4
+                    ]
+                    |> List.map (fun i -> CliType.Numeric (CliNumericType.Int32 i))
+                    |> Some
+            }
+        ]
+
+    [<TestCaseSource(nameof cases)>]
+    let ``Can evaluate C# files`` (case : TestCase) : unit =
+        let source = Assembly.getEmbeddedResourceAsString case.FileName assy
+        let image = Roslyn.compile [ source ]
+        let messages, loggerFactory = LoggerFactory.makeTest ()
+
+        let dotnetRuntimes =
+            DotnetRuntime.SelectForDll assy.Location |> ImmutableArray.CreateRange
+
+        use peImage = new MemoryStream (image)
+
+        try
+            let terminalState, terminatingThread =
+                Program.run loggerFactory (Some case.FileName) peImage dotnetRuntimes case.NativeImpls []
+
+            let realResult = RealRuntime.executeWithRealRuntime [||] image
+
+            let exitCode =
+                match terminalState.ThreadState.[terminatingThread].MethodState.EvaluationStack.Values with
+                | [] -> failwith "expected program to return a value, but it returned void"
+                | head :: _ ->
+                    match head with
+                    | EvalStackValue.Int32 i -> i
+                    | ret -> failwith $"expected program to return an int, but it returned %O{ret}"
+
+            realResult.ExitCode |> shouldEqual exitCode
+
+            exitCode |> shouldEqual case.ExpectedReturnCode
+
+            let finalVariables =
+                terminalState.ThreadState.[terminatingThread].MethodState.LocalVariables
+                |> Seq.toList
+
+            match case.LocalVariablesOfMain with
+            | None -> ()
+            | Some expected -> finalVariables |> shouldEqual expected
+        with _ ->
+            for message in messages () do
+                System.Console.Error.WriteLine $"{message}"
+
+            reraise ()
+
+    [<TestCaseSource(nameof unimplemented)>]
+    [<Explicit "not yet implemented">]
+    let ``Can evaluate C# files, unimplemented`` (case : TestCase) : unit =
+        let source = Assembly.getEmbeddedResourceAsString case.FileName assy
+        let image = Roslyn.compile [ source ]
+        let messages, loggerFactory = LoggerFactory.makeTest ()
+
+        let dotnetRuntimes =
+            DotnetRuntime.SelectForDll assy.Location |> ImmutableArray.CreateRange
+
+        use peImage = new MemoryStream (image)
+
+        try
+            let terminalState, terminatingThread =
+                Program.run loggerFactory (Some case.FileName) peImage dotnetRuntimes case.NativeImpls []
+
+            let exitCode =
+                match terminalState.ThreadState.[terminatingThread].MethodState.EvaluationStack.Values with
+                | [] -> failwith "expected program to return a value, but it returned void"
+                | head :: _ ->
+                    match head with
+                    | EvalStackValue.Int32 i -> i
+                    | ret -> failwith $"expected program to return an int, but it returned %O{ret}"
+
+            exitCode |> shouldEqual case.ExpectedReturnCode
+
+        with _ ->
+            for message in messages () do
+                System.Console.Error.WriteLine $"{message}"
+
+            reraise ()

WoofWare.PawPrint.Test/WoofWare.PawPrint.Test.fsproj

@@ -13,23 +13,28 @@
     <Compile Include="LoggerFactory.fs" />
     <Compile Include="Assembly.fs" />
     <Compile Include="Roslyn.fs" />
+    <Compile Include="RealRuntime.fs" />
     <Compile Include="TestHarness.fs"/>
-    <Compile Include="TestCases.fs" />
-    <Compile Include="TestHelloWorld.fs" />
-    <EmbeddedResource Include="sources\BasicLock.cs" />
-    <EmbeddedResource Include="sources\NoOp.cs" />
-    <EmbeddedResource Include="sources\ExceptionWithNoOpCatch.cs" />
-    <EmbeddedResource Include="sources\ExceptionWithNoOpFinally.cs" />
-    <EmbeddedResource Include="sources\TryCatchWithThrowInBody.cs" />
-    <EmbeddedResource Include="sources\ComplexTryCatch.cs" />
-    <EmbeddedResource Include="sources\TriangleNumber.cs" />
-    <EmbeddedResource Include="sources\WriteLine.cs" />
-    <EmbeddedResource Include="sources\InstaQuit.cs" />
-    <EmbeddedResource Include="sources\Threads.cs" />
-    <EmbeddedResource Include="sources\ResizeArray.cs" />
-    <EmbeddedResource Include="sources\ArgumentOrdering.cs" />
-    <EmbeddedResource Include="sources\CustomDelegate.cs" />
-    <EmbeddedResource Include="sources\Ldind.cs" />
+    <Compile Include="TestPureCases.fs" />
+    <Compile Include="TestImpureCases.fs" />
+  </ItemGroup>
+  <ItemGroup>
+    <EmbeddedResource Include="sourcesPure\BasicLock.cs" />
+    <EmbeddedResource Include="sourcesPure\NoOp.cs" />
+    <EmbeddedResource Include="sourcesPure\ExceptionWithNoOpCatch.cs" />
+    <EmbeddedResource Include="sourcesPure\ExceptionWithNoOpFinally.cs" />
+    <EmbeddedResource Include="sourcesPure\TryCatchWithThrowInBody.cs" />
+    <EmbeddedResource Include="sourcesPure\ComplexTryCatch.cs" />
+    <EmbeddedResource Include="sourcesPure\TriangleNumber.cs" />
+    <EmbeddedResource Include="sourcesPure\Threads.cs" />
+    <EmbeddedResource Include="sourcesPure\ResizeArray.cs" />
+    <EmbeddedResource Include="sourcesPure\ArgumentOrdering.cs" />
+    <EmbeddedResource Include="sourcesPure\CustomDelegate.cs" />
+    <EmbeddedResource Include="sourcesPure\Ldind.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <EmbeddedResource Include="sourcesImpure\WriteLine.cs" />
+    <EmbeddedResource Include="sourcesImpure\InstaQuit.cs" />
   </ItemGroup>
 
   <ItemGroup>
@@ -37,14 +42,13 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="FsUnit" Version="7.0.1"/>
+    <PackageReference Include="FsUnit" Version="7.1.1"/>
     <PackageReference Include="NUnit3TestAdapter" Version="5.0.0"/>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.13.0"/>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1"/>
     <PackageReference Include="NUnit" Version="4.3.2"/>
-    <PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.8.0"/>
-    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.2" />
+    <PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.14.0"/>
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.6" />
     <PackageReference Include="WoofWare.DotnetRuntimeLocator" Version="0.3.2"/>
   </ItemGroup>
-
 </Project>