diff --git a/README.md b/README.md index be9dbc5..d0940cc 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,8 @@ let ``This test fails: plain text comparison of ToString`` () = } ``` +## Updating an individual snapshot + If a snapshot is failing, add a `'` to the `expect` builder and rerun. The rerun will throw, but it will update the snapshot; then remove the `'` again to put the test back into "assert snapshot" mode. @@ -78,6 +80,51 @@ let ``Example of automatically updating`` () = } ``` +## Bulk update of snapshots + +*Warning*: when doing this, you should probably make sure your test fixture is `[]` or less parallelizable, +or the equivalent in your test runner of choice. +Otherwise, the global state used by this mechanism may interfere with other fixtures. + +You can put WoofWare.Expect into "bulk update" mode as follows: + +```fsharp +open NUnit.Framework +open WoofWare.Expect + +[] +[] +module BulkUpdateExample = + + [] + let ``Prepare to bulk-update tests`` () = + // If you don't want to enter bulk-update mode, just replace this line with a no-op `()`. + // The `updateAllSnapshots` tear-down below will simply do nothing in that case. + GlobalBuilderConfig.enterBulkUpdateMode () + + [] + let ``Update all tests`` () = + GlobalBuilderConfig.updateAllSnapshots () + + [] + let ``Snapshot 2`` () = + // this snapshot fails: the "expected" isn't even JSON! + expect { + snapshotJson "" + + return Map.ofList [ "1", "hi" ; "2", "my" ; "3", "name" ; "4", "is" ] + } + + [] + let ``Snapshot 1`` () = + // this snapshot fails: the "expected" is not equal to the "actual" + expect { + snapshotJson @"124" + return 123 + } +``` + +Observe the `OneTimeSetUp` which sets global state to enter "bulk update" mode, and the `OneTimeTearDown` which performs all the updates to rectify failures which were accumulated during this test run. # Limitations diff --git a/WoofWare.Expect.Test/BulkUpdateExample.fs b/WoofWare.Expect.Test/BulkUpdateExample.fs new file mode 100644 index 0000000..3cb9fef --- /dev/null +++ b/WoofWare.Expect.Test/BulkUpdateExample.fs @@ -0,0 +1,48 @@ +namespace WoofWare.Expect.Test + +open WoofWare.Expect +open NUnit.Framework + +[] +[] +module BulkUpdateExample = + + [] + let ``Prepare to bulk-update tests`` () = + // Uncomment the `enterBulkUpdateMode` to cause all failing tests to accumulate their results + // into a global mutable collection. + // At the end of the test run, you should then call `updateAllSnapshots ()` + // to commit these accumulated failures to the source files. + // + // When in bulk update mode, all tests will fail, to remind you to exit bulk update mode afterwards. + // + // We *strongly* recommend making these test fixtures `[]` + // or less parallelisable. + + // GlobalBuilderConfig.enterBulkUpdateMode () + () + + [] + let ``Update all tests`` () = + GlobalBuilderConfig.updateAllSnapshots () + + [] + let ``Snapshot 2`` () = + expect { + snapshotJson + @"{ + ""1"": ""hi"", + ""2"": ""my"", + ""3"": ""name"", + ""4"": ""is"" +}" + + return Map.ofList [ "1", "hi" ; "2", "my" ; "3", "name" ; "4", "is" ] + } + + [] + let ``Snapshot 1`` () = + expect { + snapshotJson @"123" + return 123 + } diff --git a/WoofWare.Expect.Test/WoofWare.Expect.Test.fsproj b/WoofWare.Expect.Test/WoofWare.Expect.Test.fsproj index c322082..ac41441 100644 --- a/WoofWare.Expect.Test/WoofWare.Expect.Test.fsproj +++ b/WoofWare.Expect.Test/WoofWare.Expect.Test.fsproj @@ -8,6 +8,7 @@ + diff --git a/WoofWare.Expect/Builder.fs b/WoofWare.Expect/Builder.fs index 54d32b1..38b5600 100644 --- a/WoofWare.Expect/Builder.fs +++ b/WoofWare.Expect/Builder.fs @@ -2,19 +2,6 @@ open System.IO open System.Runtime.CompilerServices -open System.Text.Json -open System.Text.Json.Serialization - -type private CallerInfo = - { - MemberName : string - FilePath : string - LineNumber : int - } - -type private SnapshotValue = - | BareString of string - | Json of string /// An exception indicating that a value failed to match its snapshot. exception ExpectException of Message : string @@ -23,19 +10,6 @@ exception ExpectException of Message : string /// the `snapshot` keyword multiple times. type YouHaveSuppliedMultipleSnapshots = private | NonConstructible -/// The state accumulated by the `expect` builder. You should never find yourself interacting with this type. -type ExpectState<'T> = - private - { - Snapshot : (SnapshotValue * CallerInfo) option - Actual : 'T option - } - -[] -module private Text = - let predent (c : char) (s : string) = - s.Split '\n' |> Seq.map (sprintf "%c %s" c) |> String.concat "\n" - /// Specify how the Expect computation expression treats failures. /// You probably don't want to use this directly; use the computation expression definitions /// like expect in the Builder module instead. @@ -171,71 +145,49 @@ type ExpectBuilder (mode : Mode) = /// Computation expression `Run`, which runs a `Delay`ed snapshot assertion, throwing if the assertion fails. member _.Run (f : unit -> ExpectState<'T>) : unit = - let state = f () + let state = f () |> CompletedSnapshotGeneric.make - let options = JsonFSharpOptions.Default().ToJsonSerializerOptions () - - match state.Snapshot, state.Actual with - | Some (snapshot, source), Some actual -> - let raiseError (snapshot : string) (actual : string) : unit = - match mode with - | Mode.AssertMockingSource (mockSource, line) -> + let raiseError (snapshot : string) (actual : string) : unit = + match mode with + | Mode.AssertMockingSource (mockSource, line) -> + sprintf + "snapshot mismatch! snapshot at %s:%i (%s) was:\n\n%s\n\nactual was:\n\n%s" + mockSource + line + state.Caller.MemberName + (snapshot |> Text.predent '-') + (actual |> Text.predent '+') + |> ExpectException + |> raise + | Mode.Assert -> + if GlobalBuilderConfig.bulkUpdate.Value > 0 then + GlobalBuilderConfig.registerTest state + else sprintf "snapshot mismatch! snapshot at %s:%i (%s) was:\n\n%s\n\nactual was:\n\n%s" - mockSource - line - source.MemberName + state.Caller.FilePath + state.Caller.LineNumber + state.Caller.MemberName (snapshot |> Text.predent '-') (actual |> Text.predent '+') |> ExpectException |> raise - | Mode.Assert -> - sprintf - "snapshot mismatch! snapshot at %s:%i (%s) was:\n\n%s\n\nactual was:\n\n%s" - source.FilePath - source.LineNumber - source.MemberName - (snapshot |> Text.predent '-') - (actual |> Text.predent '+') - |> ExpectException - |> raise - | Mode.Update -> - let lines = File.ReadAllLines source.FilePath - let oldContents = String.concat "\n" lines - let lines = SnapshotUpdate.updateSnapshotAtLine lines source.LineNumber actual - File.WriteAllLines (source.FilePath, lines) - failwith ("Snapshot successfully updated. Previous contents:\n" + oldContents) + | Mode.Update -> + let lines = File.ReadAllLines state.Caller.FilePath + let oldContents = String.concat "\n" lines + let lines = SnapshotUpdate.updateSnapshotAtLine lines state.Caller.LineNumber actual + File.WriteAllLines (state.Caller.FilePath, lines) + failwith ("Snapshot successfully updated. Previous contents:\n" + oldContents) - match snapshot with - | SnapshotValue.Json snapshot -> - let canonicalSnapshot = JsonDocument.Parse snapshot - - let canonicalActual = - JsonSerializer.Serialize (actual, options) |> JsonDocument.Parse - - if not (JsonElement.DeepEquals (canonicalActual.RootElement, canonicalSnapshot.RootElement)) then - raiseError (canonicalSnapshot.RootElement.ToString ()) (canonicalActual.RootElement.ToString ()) - else - match mode with - | Mode.Update -> - failwith - "Snapshot assertion passed, but we are in snapshot-updating mode. Use the `expect` builder instead of `expect'` to assert the contents of a snapshot." - | _ -> () - - | SnapshotValue.BareString snapshot -> - let actual = actual.ToString () - - if actual <> snapshot then - raiseError snapshot actual - else - match mode with - | Mode.Update -> - failwith - "Snapshot assertion passed, but we are in snapshot-updating mode. Use the `expect` builder instead of `expect'` to assert the contents of a snapshot." - | _ -> () - - | None, _ -> failwith "Must specify snapshot" - | _, None -> failwith "Must specify actual value with 'return'" + match CompletedSnapshotGeneric.passesAssertion state with + | None -> + match mode, GlobalBuilderConfig.bulkUpdate.Value with + | Mode.Update, _ + | _, 1 -> + failwith + "Snapshot assertion passed, but we are in snapshot-updating mode. Use the `expect` builder instead of `expect'` to assert the contents of a single snapshot; disable `GlobalBuilderConfig.bulkUpdate` to move back to assertion-checking mode." + | _ -> () + | Some (expected, actual) -> raiseError expected actual /// Module containing the `expect` builder. [] diff --git a/WoofWare.Expect/Config.fs b/WoofWare.Expect/Config.fs new file mode 100644 index 0000000..f99c9f3 --- /dev/null +++ b/WoofWare.Expect/Config.fs @@ -0,0 +1,52 @@ +namespace WoofWare.Expect + +open System.Threading + +/// Module holding global mutable state controlling the behaviour of WoofWare.Expect +/// when running in bulk-update mode. +[] +module GlobalBuilderConfig = + let internal bulkUpdate = ref 0 + + /// + /// Call this to make the expect builder register all tests for bulk update as it runs. + /// + /// + /// We *strongly* recommend making test fixtures Parallelizable(ParallelScope.Children) or less parallelizable (for NUnit) if you're running in bulk update mode. + /// The implied global mutable state is liable to interfere with other expect builders in other fixtures otherwise. + /// + let enterBulkUpdateMode () = + if Interlocked.Increment bulkUpdate <> 1 then + failwith + "WoofWare.Expect requires bulk updates to happen serially: for example, make the test fixture `[]` if you're using NUnit." + + let private allTests : ResizeArray = ResizeArray () + + /// + /// Clear the set of failing tests registered by any previous bulk-update runs. + /// + /// + /// + /// You probably don't need to do this, because your test runner is probably tearing down + /// anyway after the tests have failed; this is mainly here for WoofWare.Expect's own internal testing. + /// + let clearTests () = lock allTests allTests.Clear + + let internal registerTest (s : CompletedSnapshotGeneric<'T>) : unit = + let toAdd = s |> CompletedSnapshot.make + lock allTests (fun () -> allTests.Add toAdd) + + /// + /// For all tests whose failures have already been registered, + /// transform the files on disk so that the failing snapshots now pass. + /// + let updateAllSnapshots () = + let bulkUpdate' = Interlocked.Decrement bulkUpdate + + try + if bulkUpdate' = 0 then + let allTests = lock allTests (fun () -> Seq.toArray allTests) + SnapshotUpdate.updateAll allTests + + finally + clearTests () diff --git a/WoofWare.Expect/Domain.fs b/WoofWare.Expect/Domain.fs new file mode 100644 index 0000000..f92bdf4 --- /dev/null +++ b/WoofWare.Expect/Domain.fs @@ -0,0 +1,106 @@ +namespace WoofWare.Expect + +open System.Text.Json +open System.Text.Json.Serialization + +/// +/// Information about where in source code a specific snapshot is located. +/// +type CallerInfo = + internal + { + MemberName : string + FilePath : string + LineNumber : int + } + +type private SnapshotValue = + | BareString of string + | Json of string + +/// The state accumulated by the `expect` builder. You should never find yourself interacting with this type. +type ExpectState<'T> = + private + { + Snapshot : (SnapshotValue * CallerInfo) option + Actual : 'T option + } + +/// The state accumulated by the `expect` builder. You should never find yourself interacting with this type. +type internal CompletedSnapshotGeneric<'T> = + private + { + SnapshotValue : SnapshotValue + Caller : CallerInfo + Actual : 'T + } + +[] +module internal CompletedSnapshotGeneric = + let make (state : ExpectState<'T>) : CompletedSnapshotGeneric<'T> = + match state.Snapshot, state.Actual with + | Some (snapshot, source), Some actual -> + { + SnapshotValue = snapshot + Caller = source + Actual = actual + } + | None, _ -> failwith "Must specify snapshot" + | _, None -> failwith "Must specify actual value with 'return'" + + let private jsonOptions = + let options = JsonFSharpOptions.Default().ToJsonSerializerOptions () + options.AllowTrailingCommas <- true + options.WriteIndented <- true + options + + let internal replacement (s : CompletedSnapshotGeneric<'T>) = + match s.SnapshotValue with + | SnapshotValue.BareString _existing -> s.Actual.ToString () + | SnapshotValue.Json _existing -> + JsonSerializer.Serialize (s.Actual, jsonOptions) + |> JsonDocument.Parse + |> _.RootElement + |> _.ToString() + + /// Returns None if the assertion passes, or Some (expected, actual) if the assertion fails. + let internal passesAssertion (state : CompletedSnapshotGeneric<'T>) : (string * string) option = + match state.SnapshotValue with + | SnapshotValue.Json snapshot -> + let canonicalSnapshot = + try + JsonDocument.Parse snapshot |> Some + with _ -> + None + + let canonicalActual = + JsonSerializer.Serialize (state.Actual, jsonOptions) |> JsonDocument.Parse + + match canonicalSnapshot with + | None -> Some (snapshot, canonicalActual.RootElement.ToString ()) + | Some canonicalSnapshot -> + if not (JsonElement.DeepEquals (canonicalActual.RootElement, canonicalSnapshot.RootElement)) then + Some (canonicalSnapshot.RootElement.ToString (), canonicalActual.RootElement.ToString ()) + else + None + + | SnapshotValue.BareString snapshot -> + let actual = state.Actual.ToString () + + if actual = snapshot then None else Some (snapshot, actual) + +/// Represents a snapshot test that has failed and is awaiting update or report to the user. +type CompletedSnapshot = + internal + { + CallerInfo : CallerInfo + Replacement : string + } + +[] +module internal CompletedSnapshot = + let make (s : CompletedSnapshotGeneric<'T>) = + { + CallerInfo = s.Caller + Replacement = CompletedSnapshotGeneric.replacement s + } diff --git a/WoofWare.Expect/SnapshotUpdate.fs b/WoofWare.Expect/SnapshotUpdate.fs index bdda4b7..08d30e2 100644 --- a/WoofWare.Expect/SnapshotUpdate.fs +++ b/WoofWare.Expect/SnapshotUpdate.fs @@ -254,15 +254,19 @@ module internal SnapshotUpdate = /// Example usage: /// updateSnapshotAtLine [|lines-of-file|] 42 "new test output" - /// + ///
/// This will find a snapshot call on line 42 like: - /// snapshot "old value" -> snapshot @"new test output" - /// snapshot @"old value" -> snapshot @"new test output" - /// snapshot """old value""" -> snapshot @"new test output" - /// snapshot """multi - /// line""" -> snapshot """multi - /// line""" - /// snapshot "has \"\"\" in it" -> snapshot @"has """""" in it" + ///
    + ///
  • snapshot "old value" -> snapshot @"new test output"
  • + ///
  • snapshot @"old value" -> snapshot @"new test output"
  • + ///
  • snapshot """old value""" -> snapshot @"new test output"
  • + ///
  • snapshot "has \"\"\" in it" -> snapshot @"has """""" in it"
  • + ///
  • + /// snapshot """multi + /// line""" -> snapshot """multi + /// line""" + ///
  • + ///
///
let updateSnapshotAtLine (fileLines : string[]) (snapshotLine : int) (newValue : string) : string[] = match findSnapshotString fileLines snapshotLine with @@ -270,3 +274,31 @@ module internal SnapshotUpdate = Console.Error.WriteLine ("String literal to update: " + string info) updateSnapshot fileLines info newValue | None -> failwithf "Could not find string literal after snapshot at line %d" snapshotLine + + /// + /// Bulk-apply all the snapshot replacements. + /// + /// The original file contents, as an array of lines. + /// The (unsorted) line numbers of the snapshots which need to be replaced, and the replacement value for each. + /// The entire desired new contents of the file, as an array of lines. + let private updateAllLines (fileLines : string[]) (sources : (int * string) seq) : string[] = + sources + |> Seq.sortByDescending fst + |> Seq.fold (fun lines (lineNum, replacement) -> updateSnapshotAtLine lines lineNum replacement) fileLines + + /// + /// Update every failed snapshot in the input, editing the files on disk. + /// + let updateAll (sources : CompletedSnapshot seq) : unit = + sources + |> Seq.groupBy (fun csc -> csc.CallerInfo.FilePath) + |> Seq.iter (fun (callerFile, callers) -> + let contents = System.IO.File.ReadAllLines callerFile + + let sources = + callers |> Seq.map (fun csc -> csc.CallerInfo.LineNumber, csc.Replacement) + + let newContents = updateAllLines contents sources + + System.IO.File.WriteAllLines (callerFile, newContents) + ) diff --git a/WoofWare.Expect/SurfaceBaseline.txt b/WoofWare.Expect/SurfaceBaseline.txt index fef2de4..ecd5ccb 100644 --- a/WoofWare.Expect/SurfaceBaseline.txt +++ b/WoofWare.Expect/SurfaceBaseline.txt @@ -4,6 +4,10 @@ WoofWare.Expect.Builder.expect' [static property]: [read-only] WoofWare.Expect.E WoofWare.Expect.Builder.expectWithMockedFilePath [static method]: (string, int) -> WoofWare.Expect.ExpectBuilder WoofWare.Expect.Builder.get_expect [static method]: unit -> WoofWare.Expect.ExpectBuilder WoofWare.Expect.Builder.get_expect' [static method]: unit -> WoofWare.Expect.ExpectBuilder +WoofWare.Expect.CallerInfo inherit obj, implements WoofWare.Expect.CallerInfo System.IEquatable, System.Collections.IStructuralEquatable, WoofWare.Expect.CallerInfo System.IComparable, System.IComparable, System.Collections.IStructuralComparable +WoofWare.Expect.CallerInfo.Equals [method]: (WoofWare.Expect.CallerInfo, System.Collections.IEqualityComparer) -> bool +WoofWare.Expect.CompletedSnapshot inherit obj, implements WoofWare.Expect.CompletedSnapshot System.IEquatable, System.Collections.IStructuralEquatable, WoofWare.Expect.CompletedSnapshot System.IComparable, System.IComparable, System.Collections.IStructuralComparable +WoofWare.Expect.CompletedSnapshot.Equals [method]: (WoofWare.Expect.CompletedSnapshot, System.Collections.IEqualityComparer) -> bool WoofWare.Expect.ExpectBuilder inherit obj WoofWare.Expect.ExpectBuilder..ctor [constructor]: (string * int) WoofWare.Expect.ExpectBuilder..ctor [constructor]: bool @@ -23,6 +27,10 @@ WoofWare.Expect.ExpectException.Equals [method]: System.Exception -> bool WoofWare.Expect.ExpectException.Message [property]: [read-only] string WoofWare.Expect.ExpectState`1 inherit obj, implements 'T WoofWare.Expect.ExpectState System.IEquatable, System.Collections.IStructuralEquatable, 'T WoofWare.Expect.ExpectState System.IComparable, System.IComparable, System.Collections.IStructuralComparable WoofWare.Expect.ExpectState`1.Equals [method]: ('T WoofWare.Expect.ExpectState, System.Collections.IEqualityComparer) -> bool +WoofWare.Expect.GlobalBuilderConfig inherit obj +WoofWare.Expect.GlobalBuilderConfig.clearTests [static method]: unit -> unit +WoofWare.Expect.GlobalBuilderConfig.enterBulkUpdateMode [static method]: unit -> unit +WoofWare.Expect.GlobalBuilderConfig.updateAllSnapshots [static method]: unit -> unit WoofWare.Expect.Mode inherit obj, implements WoofWare.Expect.Mode System.IEquatable, System.Collections.IStructuralEquatable, WoofWare.Expect.Mode System.IComparable, System.IComparable, System.Collections.IStructuralComparable WoofWare.Expect.Mode.Equals [method]: (WoofWare.Expect.Mode, System.Collections.IEqualityComparer) -> bool WoofWare.Expect.YouHaveSuppliedMultipleSnapshots inherit obj, implements WoofWare.Expect.YouHaveSuppliedMultipleSnapshots System.IEquatable, System.Collections.IStructuralEquatable, WoofWare.Expect.YouHaveSuppliedMultipleSnapshots System.IComparable, System.IComparable, System.Collections.IStructuralComparable diff --git a/WoofWare.Expect/Text.fs b/WoofWare.Expect/Text.fs new file mode 100644 index 0000000..0f2b082 --- /dev/null +++ b/WoofWare.Expect/Text.fs @@ -0,0 +1,6 @@ +namespace WoofWare.Expect + +[] +module internal Text = + let predent (c : char) (s : string) = + s.Split '\n' |> Seq.map (sprintf "%c %s" c) |> String.concat "\n" diff --git a/WoofWare.Expect/WoofWare.Expect.fsproj b/WoofWare.Expect/WoofWare.Expect.fsproj index 45694d7..1807f97 100644 --- a/WoofWare.Expect/WoofWare.Expect.fsproj +++ b/WoofWare.Expect/WoofWare.Expect.fsproj @@ -17,7 +17,10 @@ + + + True diff --git a/WoofWare.Expect/version.json b/WoofWare.Expect/version.json index e1081d8..4bfa02e 100644 --- a/WoofWare.Expect/version.json +++ b/WoofWare.Expect/version.json @@ -1,5 +1,5 @@ { - "version": "0.2", + "version": "0.3", "publicReleaseRefSpec": [ "^refs/heads/main$" ],