Avatars and Repo avatars support storing in minio (#12516)

* Avatar support minio

* Support repo avatar minio storage

* Add missing migration

* Fix bug

* Fix test

* Add test for minio store type on avatars and repo avatars; Add documents

* Fix bug

* Fix bug

* Add back missed avatar link method

* refactor codes

* Simplify the codes

* Code improvements

* Fix lint

* Fix test mysql

* Fix test mysql

* Fix test mysql

* Fix settings

* Fix test

* fix test

* Fix bug

1 files changed, 24 insertions, 189 deletions

diff --git a/models/repo.go b/models/repo.go
index f505412e03..efdd7049de 100644
--- a/models/repo.go
+++ b/models/repo.go
@@ -7,7 +7,6 @@ package models
 
 import (
 	"context"
-	"crypto/md5"
 	"errors"
 	"fmt"
 	"html/template"
@@ -15,7 +14,6 @@ import (
 
 	// Needed for jpeg support
 	_ "image/jpeg"
-	"image/png"
 	"io/ioutil"
 	"net"
 	"net/url"
@@ -27,7 +25,6 @@ import (
 	"strings"
 	"time"
 
-	"code.gitea.io/gitea/modules/avatar"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/options"
@@ -1796,11 +1793,8 @@ func DeleteRepository(doer *User, uid, repoID int64) error {
 	}
 
 	if len(repo.Avatar) > 0 {
-		avatarPath := repo.CustomAvatarPath()
-		if com.IsExist(avatarPath) {
-			if err := util.Remove(avatarPath); err != nil {
-				return fmt.Errorf("Failed to remove %s: %v", avatarPath, err)
-			}
+		if err := storage.RepoAvatars.Delete(repo.CustomAvatarRelativePath()); err != nil {
+			return fmt.Errorf("Failed to remove %s: %v", repo.Avatar, err)
 		}
 	}
 
@@ -2239,187 +2233,6 @@ func (repo *Repository) GetUserFork(userID int64) (*Repository, error) {
 	return &forkedRepo, nil
 }
 
-// CustomAvatarPath returns repository custom avatar file path.
-func (repo *Repository) CustomAvatarPath() string {
-	// Avatar empty by default
-	if len(repo.Avatar) == 0 {
-		return ""
-	}
-	return filepath.Join(setting.RepositoryAvatarUploadPath, repo.Avatar)
-}
-
-// generateRandomAvatar generates a random avatar for repository.
-func (repo *Repository) generateRandomAvatar(e Engine) error {
-	idToString := fmt.Sprintf("%d", repo.ID)
-
-	seed := idToString
-	img, err := avatar.RandomImage([]byte(seed))
-	if err != nil {
-		return fmt.Errorf("RandomImage: %v", err)
-	}
-
-	repo.Avatar = idToString
-	if err = os.MkdirAll(filepath.Dir(repo.CustomAvatarPath()), os.ModePerm); err != nil {
-		return fmt.Errorf("MkdirAll: %v", err)
-	}
-	fw, err := os.Create(repo.CustomAvatarPath())
-	if err != nil {
-		return fmt.Errorf("Create: %v", err)
-	}
-	defer fw.Close()
-
-	if err = png.Encode(fw, img); err != nil {
-		return fmt.Errorf("Encode: %v", err)
-	}
-	log.Info("New random avatar created for repository: %d", repo.ID)
-
-	if _, err := e.ID(repo.ID).Cols("avatar").NoAutoTime().Update(repo); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// RemoveRandomAvatars removes the randomly generated avatars that were created for repositories
-func RemoveRandomAvatars(ctx context.Context) error {
-	return x.
-		Where("id > 0").BufferSize(setting.Database.IterateBufferSize).
-		Iterate(new(Repository),
-			func(idx int, bean interface{}) error {
-				repository := bean.(*Repository)
-				select {
-				case <-ctx.Done():
-					return ErrCancelledf("before random avatars removed for %s", repository.FullName())
-				default:
-				}
-				stringifiedID := strconv.FormatInt(repository.ID, 10)
-				if repository.Avatar == stringifiedID {
-					return repository.DeleteAvatar()
-				}
-				return nil
-			})
-}
-
-// RelAvatarLink returns a relative link to the repository's avatar.
-func (repo *Repository) RelAvatarLink() string {
-	return repo.relAvatarLink(x)
-}
-
-func (repo *Repository) relAvatarLink(e Engine) string {
-	// If no avatar - path is empty
-	avatarPath := repo.CustomAvatarPath()
-	if len(avatarPath) == 0 || !com.IsFile(avatarPath) {
-		switch mode := setting.RepositoryAvatarFallback; mode {
-		case "image":
-			return setting.RepositoryAvatarFallbackImage
-		case "random":
-			if err := repo.generateRandomAvatar(e); err != nil {
-				log.Error("generateRandomAvatar: %v", err)
-			}
-		default:
-			// default behaviour: do not display avatar
-			return ""
-		}
-	}
-	return setting.AppSubURL + "/repo-avatars/" + repo.Avatar
-}
-
-// AvatarLink returns a link to the repository's avatar.
-func (repo *Repository) AvatarLink() string {
-	return repo.avatarLink(x)
-}
-
-// avatarLink returns user avatar absolute link.
-func (repo *Repository) avatarLink(e Engine) string {
-	link := repo.relAvatarLink(e)
-	// link may be empty!
-	if len(link) > 0 {
-		if link[0] == '/' && link[1] != '/' {
-			return setting.AppURL + strings.TrimPrefix(link, setting.AppSubURL)[1:]
-		}
-	}
-	return link
-}
-
-// UploadAvatar saves custom avatar for repository.
-// FIXME: split uploads to different subdirs in case we have massive number of repos.
-func (repo *Repository) UploadAvatar(data []byte) error {
-	m, err := avatar.Prepare(data)
-	if err != nil {
-		return err
-	}
-
-	sess := x.NewSession()
-	defer sess.Close()
-	if err = sess.Begin(); err != nil {
-		return err
-	}
-
-	oldAvatarPath := repo.CustomAvatarPath()
-
-	// Users can upload the same image to other repo - prefix it with ID
-	// Then repo will be removed - only it avatar file will be removed
-	repo.Avatar = fmt.Sprintf("%d-%x", repo.ID, md5.Sum(data))
-	if _, err := sess.ID(repo.ID).Cols("avatar").Update(repo); err != nil {
-		return fmt.Errorf("UploadAvatar: Update repository avatar: %v", err)
-	}
-
-	if err := os.MkdirAll(setting.RepositoryAvatarUploadPath, os.ModePerm); err != nil {
-		return fmt.Errorf("UploadAvatar: Failed to create dir %s: %v", setting.RepositoryAvatarUploadPath, err)
-	}
-
-	fw, err := os.Create(repo.CustomAvatarPath())
-	if err != nil {
-		return fmt.Errorf("UploadAvatar: Create file: %v", err)
-	}
-	defer fw.Close()
-
-	if err = png.Encode(fw, *m); err != nil {
-		return fmt.Errorf("UploadAvatar: Encode png: %v", err)
-	}
-
-	if len(oldAvatarPath) > 0 && oldAvatarPath != repo.CustomAvatarPath() {
-		if err := util.Remove(oldAvatarPath); err != nil {
-			return fmt.Errorf("UploadAvatar: Failed to remove old repo avatar %s: %v", oldAvatarPath, err)
-		}
-	}
-
-	return sess.Commit()
-}
-
-// DeleteAvatar deletes the repos's custom avatar.
-func (repo *Repository) DeleteAvatar() error {
-
-	// Avatar not exists
-	if len(repo.Avatar) == 0 {
-		return nil
-	}
-
-	avatarPath := repo.CustomAvatarPath()
-	log.Trace("DeleteAvatar[%d]: %s", repo.ID, avatarPath)
-
-	sess := x.NewSession()
-	defer sess.Close()
-	if err := sess.Begin(); err != nil {
-		return err
-	}
-
-	repo.Avatar = ""
-	if _, err := sess.ID(repo.ID).Cols("avatar").Update(repo); err != nil {
-		return fmt.Errorf("DeleteAvatar: Update repository avatar: %v", err)
-	}
-
-	if _, err := os.Stat(avatarPath); err == nil {
-		if err := util.Remove(avatarPath); err != nil {
-			return fmt.Errorf("DeleteAvatar: Failed to remove %s: %v", avatarPath, err)
-		}
-	} else {
-		// // Schrodinger: file may or may not exist. See err for details.
-		log.Trace("DeleteAvatar[%d]: %v", err)
-	}
-	return sess.Commit()
-}
-
 // GetOriginalURLHostname returns the hostname of a URL or the URL
 func (repo *Repository) GetOriginalURLHostname() string {
 	u, err := url.Parse(repo.OriginalURL)
@@ -2502,3 +2315,25 @@ func DoctorUserStarNum() (err error) {
 
 	return
 }
+
+// IterateRepository iterate repositories
+func IterateRepository(f func(repo *Repository) error) error {
+	var start int
+	var batchSize = setting.Database.IterateBufferSize
+	for {
+		var repos = make([]*Repository, 0, batchSize)
+		if err := x.Limit(batchSize, start).Find(&repos); err != nil {
+			return err
+		}
+		if len(repos) == 0 {
+			return nil
+		}
+		start += len(repos)
+
+		for _, repo := range repos {
+			if err := f(repo); err != nil {
+				return err
+			}
+		}
+	}
+}