Improve LDAP synchronization efficiency (#16994)

The current LDAP sync routine has order n^2 efficiency. This change reduces this
to order n.log n.

Signed-off-by: Andrew Thornton <art27@cantab.net>

diff --git a/services/auth/source/ldap/source_search.go b/services/auth/source/ldap/source_search.go
index 71db0b770a3f6706198d75dbec64a99a2d9a2e11..9fe2443768d8a75f8bd9ff610860f96ac39e8e8f 100644 (file)
--- a/services/auth/source/ldap/source_search.go
+++ b/services/auth/source/ldap/source_search.go
@@ -26,6 +26,7 @@ type SearchResult struct {
        SSHPublicKey []string // SSH Public Key
        IsAdmin      bool     // if user is administrator
        IsRestricted bool     // if user is restricted
+       LowerName    string   // Lowername
 }
 
 func (ls *Source) sanitizedUserQuery(username string) (string, bool) {
@@ -363,6 +364,7 @@ func (ls *Source) SearchEntry(name, passwd string, directBind bool) *SearchResul
        }
 
        return &SearchResult{
+               LowerName:    strings.ToLower(username),
                Username:     username,
                Name:         firstname,
                Surname:      surname,
@@ -440,6 +442,8 @@ func (ls *Source) SearchEntries() ([]*SearchResult, error) {
                if isAttributeSSHPublicKeySet {
                        result[i].SSHPublicKey = v.GetAttributeValues(ls.AttributeSSHPublicKey)
                }
+               result[i].LowerName = strings.ToLower(result[i].Username)
+
        }
 
        return result, nil

diff --git a/services/auth/source/ldap/source_sync.go b/services/auth/source/ldap/source_sync.go
index 7e4088e57173802d26343283974e61799e2d76cf..f03e29f92049cc32b8cc4e5b7fc14a50c348eeb0 100644 (file)
--- a/services/auth/source/ldap/source_sync.go
+++ b/services/auth/source/ldap/source_sync.go
@@ -7,6 +7,7 @@ package ldap
 import (
        "context"
        "fmt"
+       "sort"
        "strings"
 
        "code.gitea.io/gitea/models"
@@ -17,7 +18,7 @@ import (
 func (source *Source) Sync(ctx context.Context, updateExisting bool) error {
        log.Trace("Doing: SyncExternalUsers[%s]", source.loginSource.Name)
 
-       var existingUsers []int64
+       var existingUsers []int
        isAttributeSSHPublicKeySet := len(strings.TrimSpace(source.AttributeSSHPublicKey)) > 0
        var sshKeysNeedUpdate bool
 
@@ -34,6 +35,10 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error {
        default:
        }
 
+       sort.Slice(users, func(i, j int) bool {
+               return users[i].LowerName < users[j].LowerName
+       })
+
        sr, err := source.SearchEntries()
        if err != nil {
                log.Error("SyncExternalUsers LDAP source failure [%s], skipped", source.loginSource.Name)
@@ -48,6 +53,12 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error {
                log.Warn("LDAP search found no entries but did not report an error. All users will be deactivated as per settings")
        }
 
+       sort.Slice(sr, func(i, j int) bool {
+               return sr[i].LowerName < sr[j].LowerName
+       })
+
+       userPos := 0
+
        for _, su := range sr {
                select {
                case <-ctx.Done():
@@ -71,12 +82,12 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error {
                }
 
                var usr *models.User
-               // Search for existing user
-               for _, du := range users {
-                       if du.LowerName == strings.ToLower(su.Username) {
-                               usr = du
-                               break
-                       }
+               for userPos < len(users) && users[userPos].LowerName < su.LowerName {
+                       userPos++
+               }
+               if userPos < len(users) && users[userPos].LowerName == su.LowerName {
+                       usr = users[userPos]
+                       existingUsers = append(existingUsers, userPos)
                }
 
                fullName := composeFullName(su.Name, su.Surname, su.Username)
@@ -85,7 +96,7 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error {
                        log.Trace("SyncExternalUsers[%s]: Creating user %s", source.loginSource.Name, su.Username)
 
                        usr = &models.User{
-                               LowerName:    strings.ToLower(su.Username),
+                               LowerName:    su.LowerName,
                                Name:         su.Username,
                                FullName:     fullName,
                                LoginType:    source.loginSource.Type,
@@ -108,8 +119,6 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error {
                                }
                        }
                } else if updateExisting {
-                       existingUsers = append(existingUsers, usr.ID)
-
                        // Synchronize SSH Public Key if that attribute is set
                        if isAttributeSSHPublicKeySet && models.SynchronizePublicKeys(usr, source.loginSource, su.SSHPublicKey) {
                                sshKeysNeedUpdate = true
@@ -161,15 +170,12 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error {
 
        // Deactivate users not present in LDAP
        if updateExisting {
-               for _, usr := range users {
-                       found := false
-                       for _, uid := range existingUsers {
-                               if usr.ID == uid {
-                                       found = true
-                                       break
-                               }
+               existPos := 0
+               for i, usr := range users {
+                       for existPos < len(existingUsers) && i > existingUsers[existPos] {
+                               existPos++
                        }
-                       if !found {
+                       if usr.IsActive && (existPos >= len(existingUsers) || i < existingUsers[existPos]) {
                                log.Trace("SyncExternalUsers[%s]: Deactivating user %s", source.loginSource.Name, usr.Name)
 
                                usr.IsActive = false