Allow multiple push mirrors (#97)

Gitea.Declarative.Lib/ConfigSchema.fs

@@ -27,14 +27,28 @@ type MergeStyle =
 
     static member toString (m : MergeStyle) = m.ToString ()
 
+[<NoComparison>]
+[<CustomEquality>]
 type PushMirror =
     {
         GitHubAddress : Uri
+        /// Gitea should always tell us a remote name, but a user in their config can't.
+        RemoteName : string option
     }
 
+    /// Equality check ignores remote names, which are not known to the user but which Gitea tracks internally.
+    override this.Equals (other : obj) : bool =
+        match other with
+        | :? PushMirror as other -> this.GitHubAddress.ToString () = other.GitHubAddress.ToString ()
+        | _ -> false
+
+    override this.GetHashCode () : int =
+        this.GitHubAddress.ToString().GetHashCode ()
+
     static member OfSerialised (s : SerialisedPushMirror) : PushMirror =
         {
             GitHubAddress = Uri s.GitHubAddress
+            RemoteName = None
         }
 
     member this.ToSerialised () : SerialisedPushMirror =
@@ -82,7 +96,7 @@ type NativeRepo =
         AllowRebase : bool option
         AllowRebaseExplicit : bool option
         AllowMergeCommits : bool option
-        Mirror : PushMirror option
+        Mirrors : PushMirror list
         ProtectedBranches : ProtectedBranch Set
         Collaborators : string Set
     }
@@ -103,7 +117,7 @@ type NativeRepo =
             AllowRebase = Some false
             AllowRebaseExplicit = Some false
             AllowMergeCommits = Some false
-            Mirror = None
+            Mirrors = []
             ProtectedBranches = Set.empty
             Collaborators = Set.empty
         }
@@ -128,7 +142,7 @@ type NativeRepo =
             AllowRebase = this.AllowRebase |> Option.orElse NativeRepo.Default.AllowRebase
             AllowRebaseExplicit = this.AllowRebaseExplicit |> Option.orElse NativeRepo.Default.AllowRebaseExplicit
             AllowMergeCommits = this.AllowMergeCommits |> Option.orElse NativeRepo.Default.AllowMergeCommits
-            Mirror = this.Mirror
+            Mirrors = this.Mirrors
             ProtectedBranches = this.ProtectedBranches // TODO should this replace null with empty?
             Collaborators = this.Collaborators
         }
@@ -149,7 +163,12 @@ type NativeRepo =
             AllowRebase = s.AllowRebase |> Option.ofNullable
             AllowRebaseExplicit = s.AllowRebaseExplicit |> Option.ofNullable
             AllowMergeCommits = s.AllowMergeCommits |> Option.ofNullable
-            Mirror = s.Mirror |> Option.ofNullable |> Option.map PushMirror.OfSerialised
+            Mirrors =
+                s.Mirrors
+                |> Option.ofObj
+                |> Option.defaultValue [||]
+                |> List.ofArray
+                |> List.map PushMirror.OfSerialised
             ProtectedBranches =
                 match s.ProtectedBranches with
                 | null -> Set.empty
@@ -179,10 +198,7 @@ type NativeRepo =
             AllowRebase = this.AllowRebase |> Option.toNullable
             AllowRebaseExplicit = this.AllowRebaseExplicit |> Option.toNullable
             AllowMergeCommits = this.AllowMergeCommits |> Option.toNullable
-            Mirror =
+            Mirrors = this.Mirrors |> List.toArray |> Array.map (fun a -> a.ToSerialised ())
-                match this.Mirror with
-                | None -> Nullable ()
-                | Some mirror -> Nullable (mirror.ToSerialised ())
             ProtectedBranches = this.ProtectedBranches |> Seq.map (fun b -> b.ToSerialised ()) |> Array.ofSeq
             Collaborators = Set.toArray this.Collaborators
         }
@@ -239,16 +255,17 @@ type Repo =
             }
             |> async.Return
         else
-            async {
+            let repoFullName = u.FullName
-                let! mirror =
-                    getAllPaginated (fun page count ->
-                        client.RepoListPushMirrors (u.Owner.LoginName, u.FullName, Some page, Some count)
-                    )
 
-                let mirror =
+            async {
-                    if mirror.Length = 0 then None
+                let owner = u.Owner
-                    elif mirror.Length = 1 then Some mirror.[0]
+
-                    else failwith "Multiple mirrors not supported yet"
+                let loginName = owner.LoginName
+
+                let! mirrors =
+                    getAllPaginated (fun page count ->
+                        client.RepoListPushMirrors (loginName, repoFullName, Some page, Some count)
+                    )
 
                 let! (branchProtections : Gitea.BranchProtection[]) =
                     client.RepoListBranchProtection (u.Owner.LoginName, u.FullName)
@@ -259,16 +276,23 @@ type Repo =
                         client.RepoListCollaborators (u.Owner.LoginName, u.FullName, Some page, Some count)
                     )
 
+                let defaultBranch = u.DefaultBranch
+
+                let collaborators =
+                    collaborators |> Seq.map (fun user -> user.LoginName) |> Set.ofSeq
+
+                let description = u.Description
+
                 return
 
                     {
-                        Description = u.Description
+                        Description = description
                         Deleted = None
                         GitHub = None
                         Native =
                             {
                                 Private = u.Private
-                                DefaultBranch = u.DefaultBranch
+                                DefaultBranch = defaultBranch
                                 IgnoreWhitespaceConflicts = u.IgnoreWhitespaceConflicts
                                 HasPullRequests = u.HasPullRequests
                                 HasProjects = u.HasProjects
@@ -281,11 +305,13 @@ type Repo =
                                 AllowRebase = u.AllowRebase
                                 AllowRebaseExplicit = u.AllowRebaseExplicit
                                 AllowMergeCommits = u.AllowMergeCommits
-                                Mirror =
+                                Mirrors =
-                                    mirror
+                                    mirrors
-                                    |> Option.map (fun m ->
+                                    |> Array.toList
+                                    |> List.map (fun m ->
                                         {
                                             GitHubAddress = Uri m.RemoteAddress
+                                            RemoteName = Some m.RemoteName
                                         }
                                     )
                                 ProtectedBranches =
@@ -302,7 +328,7 @@ type Repo =
                                         }
                                     )
                                     |> Set.ofSeq
-                                Collaborators = collaborators |> Seq.map (fun user -> user.LoginName) |> Set.ofSeq
+                                Collaborators = collaborators
                             }
                             |> Some
                     }

Gitea.Declarative.Lib/Gitea.fs

@@ -361,34 +361,134 @@ module Gitea =
                 else
                     async.Return ()
 
+            // Push mirrors
             do!
-                match desired.Mirror, actual.Mirror with
+                let desired =
-                | None, None -> async.Return ()
+                    desired.Mirrors
-                | None, Some m ->
+                    |> List.groupBy (fun m -> (m.GitHubAddress : Uri).ToString ())
-                    async { logger.LogError ("Refusing to delete push mirror for {User}:{Repo}", user, repoName) }
+                    |> Map.ofList
-                | Some desired, None ->
-                    match githubApiToken with
-                    | None ->
-                        async {
-                            logger.LogCritical (
-                                "Cannot add push mirror for {User}:{Repo} due to lack of GitHub API token",
-                                user,
-                                repoName
-                            )
-                        }
-                    | Some token ->
-                        async {
-                            logger.LogInformation ("Setting up push mirror on {User}:{Repo}", user, repoName)
-                            let pushMirrorOption = createPushMirrorOption desired.GitHubAddress token
-                            let! _ = client.RepoAddPushMirror (user, repoName, pushMirrorOption) |> Async.AwaitTask
-                            return ()
-                        }
-                | Some desired, Some actual ->
-                    if desired <> actual then
-                        async { logger.LogCritical ("Push mirror on {User}:{Repo} differs.", user, repoName) }
-                    else
-                        async.Return ()
 
+                let desired =
+                    desired
+                    |> Map.toSeq
+                    |> Seq.map (fun (name, pm) ->
+                        match pm with
+                        | [] -> failwith "LOGIC ERROR"
+                        | [ pm ] -> pm.GitHubAddress.ToString ()
+                        | _ ->
+                            failwith
+                                $"Config validation failed on repo %s{repoName}: multiple push mirrors configured for target %s{name}"
+                    )
+                    |> Set.ofSeq
+
+                let actual =
+                    actual.Mirrors
+                    |> List.groupBy (fun m -> (m.GitHubAddress : Uri).ToString ())
+                    |> Map.ofList
+
+                // If any mirror target has multiple push mirrors for it, just delete them all before continuing.
+                let deleteExisting =
+                    actual
+                    |> Map.toSeq
+                    |> Seq.choose (fun (k, vs) ->
+                        match vs with
+                        | [] -> failwith "LOGIC ERROR"
+                        | [ _ ] -> None
+                        | vs ->
+                            vs
+                            |> List.map (fun pm ->
+                                async {
+                                    logger.LogWarning (
+                                        "Multiple push mirrors on repo {Owner}/{RepoName} for target {PushMirrorTarget} found. Deleting them all before recreating.",
+                                        user,
+                                        repoName,
+                                        k
+                                    )
+
+                                    let! ct = Async.CancellationToken
+                                    // sigh, domain model - it's *such* a faff to represent this correctly though
+                                    do!
+                                        client.RepoDeletePushMirror (user, repoName, Option.get pm.RemoteName)
+                                        |> Async.AwaitTask
+                                }
+                            )
+                            |> Async.Sequential
+                            |> Async.map (Array.iter id)
+                            |> Some
+                    )
+                    |> Seq.toList
+
+                let actual =
+                    match deleteExisting with
+                    | [] -> actual
+                    | _ -> Map.empty
+
+                let distinctActual = actual.Keys |> Set.ofSeq
+                let presentButNotDesired = Set.difference distinctActual desired
+                let desiredButNotPresent = Set.difference desired distinctActual
+
+                let deleteUndesired =
+                    presentButNotDesired
+                    |> Seq.map (fun toDelete ->
+                        logger.LogWarning (
+                            "Deleting push mirror on repo {Owner}/{RepoName} for target {PushMirrorTarget}",
+                            user,
+                            repoName,
+                            toDelete
+                        )
+
+                        let toDelete = actual.[toDelete]
+
+                        toDelete
+                        |> Seq.map (fun pm ->
+                            async {
+                                let! ct = Async.CancellationToken
+
+                                do!
+                                    client.RepoDeletePushMirror (user, repoName, Option.get pm.RemoteName)
+                                    |> Async.AwaitTask
+                            }
+                        )
+                        |> Async.Sequential
+                        |> Async.map (Array.iter id)
+                    )
+                    |> Seq.toList
+
+                let addDesired =
+                    desiredButNotPresent
+                    |> Seq.map (fun toAdd ->
+                        match githubApiToken with
+                        | None ->
+                            async {
+                                logger.LogCritical (
+                                    "Cannot add push mirror for {User}:{Repo} due to lack of GitHub API token",
+                                    user,
+                                    repoName
+                                )
+                            }
+                        | Some token ->
+                            async {
+                                logger.LogInformation ("Setting up push mirror on {User}:{Repo}", user, repoName)
+                                let! ct = Async.CancellationToken
+                                let pushMirrorOption = createPushMirrorOption (Uri toAdd) token
+
+                                let! _ = client.RepoAddPushMirror (user, repoName, pushMirrorOption) |> Async.AwaitTask
+
+                                return ()
+                            }
+                    )
+                    |> Seq.toList
+
+                if deleteExisting.IsEmpty && deleteUndesired.IsEmpty && addDesired.IsEmpty then
+                    async.Return ()
+                else
+                    async {
+                        do! deleteExisting |> Async.Sequential |> Async.map (Array.iter id)
+                        do! deleteUndesired |> Async.Sequential |> Async.map (Array.iter id)
+                        do! addDesired |> Async.Sequential |> Async.map (Array.iter id)
+                    }
+
+            // Collaborators
             do!
                 let desiredButNotPresent = Set.difference desired.Collaborators actual.Collaborators
                 let presentButNotDesired = Set.difference actual.Collaborators desired.Collaborators
@@ -547,30 +647,44 @@ module Gitea =
                             | Choice2Of2 e -> raise (AggregateException ($"Error creating {user}:{r}", e))
                             | Choice1Of2 _ -> ()
 
-                            match native.Mirror, githubApiToken with
+                            match native.Mirrors, githubApiToken with
-                            | None, _ -> ()
+                            | [], _ -> ()
-                            | Some mirror, None -> failwith "Cannot push to GitHub mirror with an API key"
+                            | _ :: _, None -> failwith "Cannot push to GitHub mirror without an API key"
-                            | Some mirror, Some token ->
+                            | mirrors, Some token ->
                                 logger.LogInformation ("Setting up push mirror for {User}:{Repo}", user, r)
-                                let options = Gitea.CreatePushMirrorOption ()
-                                options.SyncOnCommit <- Some true
-                                options.RemoteAddress <- (mirror.GitHubAddress : Uri).ToString ()
-                                options.RemoteUsername <- token
-                                options.RemotePassword <- token
-                                options.Interval <- "8h0m0s"
 
-                                let! mirrors =
+                                let! actualMirrors =
                                     getAllPaginated (fun page count ->
                                         client.RepoListPushMirrors (user, r, Some page, Some count)
                                     )
 
-                                match mirrors |> Array.tryFind (fun m -> m.RemoteAddress = options.RemoteAddress) with
+                                do!
-                                | None ->
+                                    mirrors
-                                    let! _ = client.RepoAddPushMirror (user, r, options) |> Async.AwaitTask
+                                    |> List.map (fun mirror ->
-                                    ()
+                                        let options = Gitea.CreatePushMirrorOption ()
-                                | Some existing ->
+                                        options.SyncOnCommit <- Some true
-                                    if existing.SyncOnCommit <> Some true then
+                                        options.RemoteAddress <- (mirror.GitHubAddress : Uri).ToString ()
-                                        failwith $"sync on commit should have been true for {user}:{r}"
+                                        options.RemoteUsername <- token
+                                        options.RemotePassword <- token
+                                        options.Interval <- "8h0m0s"
+
+                                        async {
+                                            match
+                                                actualMirrors
+                                                |> Array.tryFind (fun m -> m.RemoteAddress = options.RemoteAddress)
+                                            with
+                                            | None ->
+                                                let! _ =
+                                                    client.RepoAddPushMirror (user, r, options) |> Async.AwaitTask
+
+                                                ()
+                                            | Some existing ->
+                                                if existing.SyncOnCommit <> Some true then
+                                                    failwith $"sync on commit should have been true for {user}:{r}"
+                                        }
+                                    )
+                                    |> Async.Sequential
+                                    |> Async.map (Array.iter id)
 
                                 ()
                         | Some github, None ->

Gitea.Declarative.Lib/GiteaConfig.schema.json

@@ -188,13 +188,12 @@
           "type": "boolean",
           "description": "either `true` to allow merging pull requests with a merge commit, or `false` to prevent merging pull requests with merge commits."
         },
-        "mirror": {
+        "mirrors": {
-          "description": "Configure a GitHub push mirror to sync this repo to",
+          "type": "array",
-          "oneOf": [
+          "description": "Configure GitHub push mirrors to sync this repo to",
-            {
+          "items": {
-              "$ref": "#/definitions/SerialisedPushMirror"
+            "$ref": "#/definitions/SerialisedPushMirror"
-            }
+          }
-          ]
         },
         "protectedBranches": {
           "type": [

Gitea.Declarative.Lib/SerialisedConfigSchema.fs

@@ -81,8 +81,8 @@ type internal SerialisedNativeRepo =
         [<Description "either `true` to allow merging pull requests with a merge commit, or `false` to prevent merging pull requests with merge commits.">]
         AllowMergeCommits : Nullable<bool>
         [<JsonProperty(Required = Required.DisallowNull)>]
-        [<Description "Configure a GitHub push mirror to sync this repo to">]
+        [<Description "Configure GitHub push mirrors to sync this repo to">]
-        Mirror : Nullable<SerialisedPushMirror>
+        Mirrors : SerialisedPushMirror[]
         [<JsonProperty(Required = Required.Default)>]
         [<Description "Protected branch configuration">]
         ProtectedBranches : SerialisedProtectedBranch array

Gitea.Declarative.Test/GiteaConfig.json

@@ -38,9 +38,11 @@
                 "native": {
                     "defaultBranch": "main",
                     "private": false,
-                    "mirror": {
+                    "mirrors": [
-                        "gitHubAddress": "https://github.com/MyName/repo-name-3"
+                        {
-                    }
+                            "gitHubAddress": "https://github.com/MyName/repo-name-3"
+                        }
+                    ]
                 }
             },
             "new-repo-mirrored-with-branches": {
@@ -48,9 +50,11 @@
                 "native": {
                     "defaultBranch": "main",
                     "private": false,
-                    "mirror": {
+                    "mirrors": [
-                        "gitHubAddress": "https://github.com/MyName/repo-name-3"
+                        {
-                    },
+                            "gitHubAddress": "https://github.com/MyName/repo-name-3"
+                        }
+                    ],
                     "protectedBranches": [
                         {
                             "branchName": "main"
@@ -63,9 +67,11 @@
                 "native": {
                     "defaultBranch": "main",
                     "private": false,
-                    "mirror": {
+                    "mirrors": [
-                        "gitHubAddress": "https://github.com/MyName/repo-name-3"
+                        {
-                    },
+                            "gitHubAddress": "https://github.com/MyName/repo-name-3"
+                        }
+                    ],
                     "protectedBranches": [
                         {
                             "branchName": "main",
@@ -83,9 +89,11 @@
                 "native": {
                     "defaultBranch": "main",
                     "private": false,
-                    "mirror": {
+                    "mirrors": [
-                        "gitHubAddress": "https://github.com/MyName/repo-name-3"
+                        {
-                    },
+                            "gitHubAddress": "https://github.com/MyName/repo-name-3"
+                        }
+                    ],
                     "protectedBranches": [
                         {
                             "branchName": "main",

Gitea.Declarative.Test/TestJsonSchema.fs

@@ -46,15 +46,14 @@ module TestSchema =
         let schema = reader.ReadToEnd ()
         schema.Contains "SerialisedGiteaConfig" |> shouldEqual true
 
-    [<Test>]
+    let schemaFile : Lazy<FileInfo> =
-    [<Explicit "Run this to regenerate the schema file">]
+        lazy
-    let ``Update schema file`` () =
-        let schemaFile =
             Assembly.GetExecutingAssembly().Location
             |> FileInfo
             |> fun fi -> fi.Directory
             |> Utils.findFileAbove "Gitea.Declarative.Lib/GiteaConfig.schema.json"
 
+    let computeSchema () =
         let settings = JsonSchemaGeneratorSettings ()
 
         settings.SerializerSettings <-
@@ -78,4 +77,16 @@ module TestSchema =
             schema.RequiredProperties.Add "native"
             serialisedRepoSchema.OneOf.Add schema
 
-        File.WriteAllText (schemaFile.FullName, schema.ToJson ())
+        schema
+
+    [<Test>]
+    let ``Schema hasn't changed`` () =
+        let computed = computeSchema () |> fun x -> x.ToJson ()
+        let actual = File.ReadAllText (schemaFile.Force().FullName)
+        computed |> shouldEqual actual
+
+    [<Test>]
+    [<Explicit "Run this to regenerate the schema file">]
+    let ``Update schema file`` () =
+        let schema = computeSchema ()
+        File.WriteAllText (schemaFile.Force().FullName, schema.ToJson ())