From 9a12abe0cfddb77838f1252ba7ab173f875a3239 Mon Sep 17 00:00:00 2001 From: Patrick Stevens Date: Mon, 7 Aug 2023 12:13:53 +0100 Subject: [PATCH] Subcommands (#62) --- Gitea.Declarative.Lib/ConfigSchema.fs | 9 + .../Gitea.Declarative.Lib.fsproj | 2 +- Gitea.Declarative.Test/TestJsonSchema.fs | 9 + Gitea.Declarative/ArgsCrate.fs | 20 +++ Gitea.Declarative/Gitea.Declarative.fsproj | 7 + Gitea.Declarative/OutputSchema.fs | 47 ++++++ Gitea.Declarative/Program.fs | 159 +++++++----------- Gitea.Declarative/Reconcile.fs | 116 +++++++++++++ Gitea.Declarative/Result.fs | 9 + Gitea.Declarative/Verify.fs | 62 +++++++ README.md | 4 +- 11 files changed, 340 insertions(+), 104 deletions(-) create mode 100644 Gitea.Declarative/ArgsCrate.fs create mode 100644 Gitea.Declarative/OutputSchema.fs create mode 100644 Gitea.Declarative/Reconcile.fs create mode 100644 Gitea.Declarative/Result.fs create mode 100644 Gitea.Declarative/Verify.fs diff --git a/Gitea.Declarative.Lib/ConfigSchema.fs b/Gitea.Declarative.Lib/ConfigSchema.fs index ecfbba7..17ed3d8 100644 --- a/Gitea.Declarative.Lib/ConfigSchema.fs +++ b/Gitea.Declarative.Lib/ConfigSchema.fs @@ -447,3 +447,12 @@ module GiteaConfig = JsonConvert.DeserializeObject s |> GiteaConfig.OfSerialised + + let getSchema () : Stream = + let resource = "Gitea.Declarative.Lib.GiteaConfig.schema.json" + let assembly = System.Reflection.Assembly.GetExecutingAssembly () + let stream = assembly.GetManifestResourceStream resource + + match stream with + | null -> failwithf "The resource %s was not found. This is a bug in the tool." resource + | stream -> stream diff --git a/Gitea.Declarative.Lib/Gitea.Declarative.Lib.fsproj b/Gitea.Declarative.Lib/Gitea.Declarative.Lib.fsproj index ca4b094..e8c1f57 100644 --- a/Gitea.Declarative.Lib/Gitea.Declarative.Lib.fsproj +++ b/Gitea.Declarative.Lib/Gitea.Declarative.Lib.fsproj @@ -29,7 +29,7 @@ - + diff --git a/Gitea.Declarative.Test/TestJsonSchema.fs b/Gitea.Declarative.Test/TestJsonSchema.fs index 028b70a..fc84708 100644 --- a/Gitea.Declarative.Test/TestJsonSchema.fs +++ b/Gitea.Declarative.Test/TestJsonSchema.fs @@ -18,6 +18,8 @@ module TestSchema = let ``Example conforms to schema`` () = let executing = Assembly.GetExecutingAssembly().Location |> FileInfo + // We choose to refer to the path specifically here, so that the "Update" functionality + // below can't be broken by an undetected file rename. let schemaFile = Utils.findFileAbove "Gitea.Declarative.Lib/GiteaConfig.schema.json" executing.Directory @@ -37,6 +39,13 @@ module TestSchema = let jsonFile = Utils.findFileAbove "GiteaConfig.json" executing.Directory GiteaConfig.get jsonFile |> ignore + [] + let ``Schema can be output`` () = + use schema = GiteaConfig.getSchema () + let reader = new StreamReader (schema) + let schema = reader.ReadToEnd () + schema.Contains "SerialisedGiteaConfig" |> shouldEqual true + [] [] let ``Update schema file`` () = diff --git a/Gitea.Declarative/ArgsCrate.fs b/Gitea.Declarative/ArgsCrate.fs new file mode 100644 index 0000000..1dcffa5 --- /dev/null +++ b/Gitea.Declarative/ArgsCrate.fs @@ -0,0 +1,20 @@ +namespace Gitea.Declarative + +open Argu + +type ArgsEvaluator<'ret> = + abstract Eval<'a, 'b when 'b :> IArgParserTemplate> : + (ParseResults<'b> -> Result<'a, ArguParseException>) -> ('a -> Async) -> 'ret + +type ArgsCrate = + abstract Apply<'ret> : ArgsEvaluator<'ret> -> 'ret + +[] +module ArgsCrate = + let make<'a, 'b when 'b :> IArgParserTemplate> + (ofResult : ParseResults<'b> -> Result<'a, ArguParseException>) + (run : 'a -> Async) + = + { new ArgsCrate with + member _.Apply e = e.Eval ofResult run + } diff --git a/Gitea.Declarative/Gitea.Declarative.fsproj b/Gitea.Declarative/Gitea.Declarative.fsproj index 659abcb..4c2a6fa 100644 --- a/Gitea.Declarative/Gitea.Declarative.fsproj +++ b/Gitea.Declarative/Gitea.Declarative.fsproj @@ -16,6 +16,11 @@ + + + + + @@ -27,5 +32,7 @@ + + diff --git a/Gitea.Declarative/OutputSchema.fs b/Gitea.Declarative/OutputSchema.fs new file mode 100644 index 0000000..eec18c7 --- /dev/null +++ b/Gitea.Declarative/OutputSchema.fs @@ -0,0 +1,47 @@ +namespace Gitea.Declarative + +open System.IO +open Argu + +type OutputSchemaArgsFragment = + | Output of string + + interface IArgParserTemplate with + member s.Usage = + match s with + | Output _ -> "path to the file to be written (or overwritten, if it already exists), instead of stdout" + +type OutputSchemaArgs = + { + Output : FileInfo option + } + + static member OfParse + (parsed : ParseResults) + : Result + = + try + { + Output = parsed.TryGetResult OutputSchemaArgsFragment.Output |> Option.map FileInfo + } + |> Ok + with :? ArguParseException as e -> + Error e + +[] +module OutputSchema = + + let run (args : OutputSchemaArgs) : Async = + async { + use stream = GiteaConfig.getSchema () + + match args.Output with + | None -> + let reader = new StreamReader (stream) + System.Console.WriteLine (reader.ReadToEnd ()) + | Some output -> + use output = output.OpenWrite () + stream.CopyTo output + + return 0 + } diff --git a/Gitea.Declarative/Program.fs b/Gitea.Declarative/Program.fs index 4e4c6e6..9396af9 100644 --- a/Gitea.Declarative/Program.fs +++ b/Gitea.Declarative/Program.fs @@ -1,42 +1,7 @@ namespace Gitea.Declarative -open System -open System.IO -open System.Net.Http -open Microsoft.Extensions.Logging -open Microsoft.Extensions.Logging.Console -open Microsoft.Extensions.Options open Argu -type ArgsFragments = - | [] Config_File of string - | [] Gitea_Host of string - | [] Gitea_Admin_Api_Token of - string - | [] GitHub_Api_Token of string - | Dry_Run - - interface IArgParserTemplate with - member s.Usage = - match s with - | Config_File _ -> - "a config file, JSON, conforming to GiteaConfig.schema.json, specifying the desired Gitea configuration" - | Gitea_Host _ -> "the Gitea host, e.g. https://gitea.mydomain.com" - | Gitea_Admin_Api_Token _ -> - "a Gitea admin user's API token; can be read from the environment variable GITEA_ADMIN_API_TOKEN" - | GitHub_Api_Token _ -> - "a GitHub API token with read access to every desired sync-from-GitHub repo; can be read from the environment variable GITHUB_API_TOKEN" - | Dry_Run _ -> "don't actually perform the reconciliation" - -type Args = - { - ConfigFile : FileInfo - GiteaHost : Uri - GiteaAdminApiToken : string - GitHubApiToken : string option - DryRun : bool - } - module Program = let printUserErrors (m : Map>) = @@ -46,88 +11,78 @@ module Program = m |> Map.iter (fun (User u) errMap -> errMap |> Map.iter (fun (RepoName r) err -> printfn $"%s{u}: %s{r}: {err}")) - let getUserInput (s : string) : string = - Console.Write s - Console.ReadLine () + let subcommands = + [| + "reconcile", + ("Reconcile a remote Gitea server with a declarative configuration", + ArgsCrate.make RunArgs.OfParse Reconcile.run) + + "output-schema", + ("Output a schema you can use to verify the `reconcile` config file", + ArgsCrate.make OutputSchemaArgs.OfParse OutputSchema.run) + + "verify", ("Verify a `reconcile` configuration file", ArgsCrate.make VerifyArgs.OfParse Verify.run) + |] + |> Map.ofArray [] let main argv = - let parser = ArgumentParser.Create () - let reader = ConfigurationReader.FromEnvironmentVariables () + // It looks like Argu doesn't really support the combination of subcommands and read-from-env-vars, so we just + // roll our own. - let parsed = - try - parser.Parse (argv, reader, raiseOnUsage = true) |> Some - with :? ArguParseException as e -> - eprintfn "%s" e.Message - None + match Array.tryHead argv with + | None + | Some "--help" -> + subcommands.Keys + |> String.concat "," + |> eprintfn "Subcommands (try each with `--help`): %s" - match parsed with - | None -> 127 - | Some parsed -> + 127 - let args = - { - ConfigFile = parsed.GetResult ArgsFragments.Config_File |> FileInfo - GiteaHost = parsed.GetResult ArgsFragments.Gitea_Host |> Uri - GiteaAdminApiToken = parsed.GetResult ArgsFragments.Gitea_Admin_Api_Token - GitHubApiToken = parsed.TryGetResult ArgsFragments.GitHub_Api_Token - DryRun = parsed.TryGetResult ArgsFragments.Dry_Run |> Option.isSome - } + | Some commandName -> - let config = GiteaConfig.get args.ConfigFile + match Map.tryFind commandName subcommands with + | None -> + subcommands.Keys + |> String.concat "," + |> eprintfn "Unrecognised command '%s'. Subcommands (try each with `--help`): %s" commandName - let options = - let options = ConsoleLoggerOptions () + 127 - { new IOptionsMonitor with - member _.Get _ = options - member _.CurrentValue = options + | Some (_help, command) -> - member _.OnChange _ = - { new IDisposable with - member _.Dispose () = () - } - } + let argv = Array.tail argv + let config = ConfigurationReader.FromEnvironmentVariables () - use loggerProvider = new ConsoleLoggerProvider (options) - let logger = loggerProvider.CreateLogger "Gitea.Declarative" + { new ArgsEvaluator<_> with + member _.Eval<'a, 'b when 'b :> IArgParserTemplate> + (ofParseResult : ParseResults<'b> -> Result<'a, _>) + run + = + let parser = ArgumentParser.Create<'b> () - use client = new HttpClient () - client.BaseAddress <- args.GiteaHost - client.DefaultRequestHeaders.Add ("Authorization", $"token {args.GiteaAdminApiToken}") + let parsed = + try + parser.Parse (argv, config, raiseOnUsage = true) |> Some + with :? ArguParseException as e -> + e.Message.Replace ("Gitea.Declarative ", sprintf "Gitea.Declarative %s " commandName) + |> eprintfn "%s" - let client = Gitea.Client client |> IGiteaClient.fromReal + None - task { - logger.LogInformation "Checking users..." - let! userErrors = Gitea.checkUsers config client + match parsed with + | None -> Error 127 + | Some parsed -> - match userErrors, args.DryRun with - | Ok (), _ -> () - | Error errors, false -> do! Gitea.reconcileUserErrors logger getUserInput client errors - | Error errors, true -> - logger.LogError ( - "Differences encountered in user configuration, but not reconciling them due to --dry-run. Errors may occur while checking repo configuration. {UserErrors}", - errors - ) + match ofParseResult parsed with + | Error e -> + e.Message.Replace ("Gitea.Declarative ", sprintf "Gitea.Declarative %s " commandName) + |> eprintfn "%s" - logger.LogInformation "Checking repos..." - let! repoErrors = Gitea.checkRepos logger config client + Error 127 + | Ok args -> - match repoErrors, args.DryRun with - | Ok (), _ -> () - | Error errors, false -> do! Gitea.reconcileRepoErrors logger client args.GitHubApiToken errors - | Error errors, true -> - logger.LogError ( - "Differences encountered in repo configuration, but not reconciling them due to --dry-run. {RepoErrors}", - errors - ) - - match userErrors, repoErrors with - | Ok (), Ok () -> return 0 - | Ok (), Error _ -> return 1 - | Error _, Ok () -> return 2 - | Error _, Error _ -> return 3 + run args |> Ok } - |> fun t -> t.Result + |> command.Apply + |> Result.cata Async.RunSynchronously id diff --git a/Gitea.Declarative/Reconcile.fs b/Gitea.Declarative/Reconcile.fs new file mode 100644 index 0000000..492fa7b --- /dev/null +++ b/Gitea.Declarative/Reconcile.fs @@ -0,0 +1,116 @@ +namespace Gitea.Declarative + +open System +open System.IO +open System.Net.Http +open Argu +open Microsoft.Extensions.Logging.Console +open Microsoft.Extensions.Options +open Microsoft.Extensions.Logging + +type RunArgsFragment = + | [] Config_File of string + | [] Gitea_Host of string + | [] Gitea_Admin_Api_Token of + string + | [] GitHub_Api_Token of string + | Dry_Run + + interface IArgParserTemplate with + member s.Usage = + match s with + | Config_File _ -> + "a config file, JSON, conforming to GiteaConfig.schema.json, specifying the desired Gitea configuration" + | Gitea_Host _ -> "the Gitea host, e.g. https://gitea.mydomain.com" + | Gitea_Admin_Api_Token _ -> + "a Gitea admin user's API token; can be read from the environment variable GITEA_ADMIN_API_TOKEN" + | GitHub_Api_Token _ -> + "a GitHub API token with read access to every desired sync-from-GitHub repo; can be read from the environment variable GITHUB_API_TOKEN" + | Dry_Run _ -> "don't actually perform the reconciliation" + +type RunArgs = + { + ConfigFile : FileInfo + GiteaHost : Uri + GiteaAdminApiToken : string + GitHubApiToken : string option + DryRun : bool + } + + static member OfParse (parsed : ParseResults) : Result = + try + { + ConfigFile = parsed.GetResult RunArgsFragment.Config_File |> FileInfo + GiteaHost = parsed.GetResult RunArgsFragment.Gitea_Host |> Uri + GiteaAdminApiToken = parsed.GetResult RunArgsFragment.Gitea_Admin_Api_Token + GitHubApiToken = parsed.TryGetResult RunArgsFragment.GitHub_Api_Token + DryRun = parsed.TryGetResult RunArgsFragment.Dry_Run |> Option.isSome + } + |> Ok + with :? ArguParseException as e -> + Error e + +[] +module Reconcile = + + let private getUserInput (s : string) : string = + Console.Write s + Console.ReadLine () + + let run (args : RunArgs) : Async = + + let config = GiteaConfig.get args.ConfigFile + + let options = + let options = ConsoleLoggerOptions () + + { new IOptionsMonitor with + member _.Get _ = options + member _.CurrentValue = options + + member _.OnChange _ = + { new IDisposable with + member _.Dispose () = () + } + } + + async { + use loggerProvider = new ConsoleLoggerProvider (options) + let logger = loggerProvider.CreateLogger "Gitea.Declarative" + + use client = new HttpClient () + client.BaseAddress <- args.GiteaHost + client.DefaultRequestHeaders.Add ("Authorization", $"token {args.GiteaAdminApiToken}") + + let client = Gitea.Client client |> IGiteaClient.fromReal + + logger.LogInformation "Checking users..." + let! userErrors = Gitea.checkUsers config client + + match userErrors, args.DryRun with + | Ok (), _ -> () + | Error errors, false -> do! Gitea.reconcileUserErrors logger getUserInput client errors + | Error errors, true -> + logger.LogError ( + "Differences encountered in user configuration, but not reconciling them due to --dry-run. Errors may occur while checking repo configuration. {UserErrors}", + errors + ) + + logger.LogInformation "Checking repos..." + let! repoErrors = Gitea.checkRepos logger config client + + match repoErrors, args.DryRun with + | Ok (), _ -> () + | Error errors, false -> do! Gitea.reconcileRepoErrors logger client args.GitHubApiToken errors + | Error errors, true -> + logger.LogError ( + "Differences encountered in repo configuration, but not reconciling them due to --dry-run. {RepoErrors}", + errors + ) + + match userErrors, repoErrors with + | Ok (), Ok () -> return 0 + | Ok (), Error _ -> return 1 + | Error _, Ok () -> return 2 + | Error _, Error _ -> return 3 + } diff --git a/Gitea.Declarative/Result.fs b/Gitea.Declarative/Result.fs new file mode 100644 index 0000000..8c33b4b --- /dev/null +++ b/Gitea.Declarative/Result.fs @@ -0,0 +1,9 @@ +namespace Gitea.Declarative + +[] +module Result = + + let cata<'ok, 'err, 'result> onOk onError (r : Result<'ok, 'err>) : 'result = + match r with + | Ok ok -> onOk ok + | Error e -> onError e diff --git a/Gitea.Declarative/Verify.fs b/Gitea.Declarative/Verify.fs new file mode 100644 index 0000000..ce1a41b --- /dev/null +++ b/Gitea.Declarative/Verify.fs @@ -0,0 +1,62 @@ +namespace Gitea.Declarative + +open System +open System.IO +open Argu +open NJsonSchema +open NJsonSchema.Validation + +type VerifyArgsFragment = + | [] Input of string + + interface IArgParserTemplate with + member s.Usage = + match s with + | Input _ -> "path to the file to be verified, or the literal '-' to read from stdin" + +type VerifyArgs = + | File of FileInfo + | Stdin + + static member OfParse (parsed : ParseResults) : Result = + let input = + try + parsed.GetResult VerifyArgsFragment.Input |> Ok + with :? ArguParseException as e -> + Error e + + input + |> Result.map (fun input -> + if input = "-" then + VerifyArgs.Stdin + else + VerifyArgs.File (FileInfo input) + ) + +[] +module Verify = + let run (args : VerifyArgs) : Async = + async { + let validator = JsonSchemaValidator () + use schema = GiteaConfig.getSchema () + let! ct = Async.CancellationToken + let! schema = JsonSchema.FromJsonAsync (schema, ct) |> Async.AwaitTask + + use jsonStream = + match args with + | VerifyArgs.Stdin -> Console.OpenStandardInput () + | VerifyArgs.File f -> f.OpenRead () + + let reader = new StreamReader (jsonStream) + let! json = reader.ReadToEndAsync ct |> Async.AwaitTask + + let errors = validator.Validate (json, schema) + + if errors.Count = 0 then + return 0 + else + for error in errors do + Console.Error.WriteLine (sprintf "Error: %+A" error) + + return 1 + } diff --git a/README.md b/README.md index be3fe61..13446a7 100644 --- a/README.md +++ b/README.md @@ -11,16 +11,18 @@ This is a small project to allow you to specify a [Gitea](https://github.com/go- * Optional branch protection rules * Pull request configuration (e.g. whether rebase-merges are allowed, etc) * Collaborators +* Reconciliation of differences between configuration and reality in the above * Deletion of repositories, guarded by the `"deleted": true` configuration # Arguments Run with the `--help` argument for a full listing. The main argument you provide is a JSON configuration file, which should conform to [the schema](./Gitea.Declarative.Lib/GiteaConfig.schema.json); there is [an example](./Gitea.Declarative.Test/GiteaConfig.json) in the tests. +(You can call `dotnet-gitea-declarative verify $CONFIG_PATH` to verify against the schema, or `dotnet-gitea-declarative output-schema` to output the schema for local tooling to consume.) # How to build and run -* With Nix: `nix run github:Smaug123/dotnet-gitea-declarative -- --help`. +* With Nix: `nix run github:Smaug123/dotnet-gitea-declarative -- reconcile --help`. * From source: clone the repository, and `dotnet run`. # Demos