Update migrated repositories' issues/comments/prs poster id if user has a github external user saved (#7751)

* update migrated issues/comments when login as github

* add get userid when migrating or login with github oauth2

* fix lint

* add migrations for repository service type

* fix build

* remove unnecessary dependencies on migrations

* add cron task to update migrations poster ids and fix posterid when migrating

* fix lint

* fix lint

* improve code

* fix lint

* improve code

* replace releases publish id to actual author id

* fix import

* fix bug

* fix lint

* fix rawdata definition

* fix some bugs

* fix error message

7 files changed, 272 insertions, 29 deletions

diff --git a/models/external_login_user.go b/models/external_login_user.go
index 21a3cbbd31..5058fd1b4b 100644
--- a/models/external_login_user.go
+++ b/models/external_login_user.go
@@ -4,13 +4,34 @@
 
 package models
 
-import "github.com/markbates/goth"
+import (
+	"time"
+
+	"code.gitea.io/gitea/modules/structs"
+
+	"github.com/markbates/goth"
+	"xorm.io/builder"
+)
 
 // ExternalLoginUser makes the connecting between some existing user and additional external login sources
 type ExternalLoginUser struct {
-	ExternalID    string `xorm:"pk NOT NULL"`
-	UserID        int64  `xorm:"INDEX NOT NULL"`
-	LoginSourceID int64  `xorm:"pk NOT NULL"`
+	ExternalID        string                 `xorm:"pk NOT NULL"`
+	UserID            int64                  `xorm:"INDEX NOT NULL"`
+	LoginSourceID     int64                  `xorm:"pk NOT NULL"`
+	RawData           map[string]interface{} `xorm:"TEXT JSON"`
+	Provider          string                 `xorm:"index VARCHAR(25)"`
+	Email             string
+	Name              string
+	FirstName         string
+	LastName          string
+	NickName          string
+	Description       string
+	AvatarURL         string
+	Location          string
+	AccessToken       string
+	AccessTokenSecret string
+	RefreshToken      string
+	ExpiresAt         time.Time
 }
 
 // GetExternalLogin checks if a externalID in loginSourceID scope already exists
@@ -32,23 +53,15 @@ func ListAccountLinks(user *User) ([]*ExternalLoginUser, error) {
 	return externalAccounts, nil
 }
 
-// LinkAccountToUser link the gothUser to the user
-func LinkAccountToUser(user *User, gothUser goth.User) error {
-	loginSource, err := GetActiveOAuth2LoginSourceByName(gothUser.Provider)
-	if err != nil {
-		return err
-	}
-
-	externalLoginUser := &ExternalLoginUser{
-		ExternalID:    gothUser.UserID,
-		UserID:        user.ID,
-		LoginSourceID: loginSource.ID,
-	}
-	has, err := x.Get(externalLoginUser)
+// LinkExternalToUser link the external user to the user
+func LinkExternalToUser(user *User, externalLoginUser *ExternalLoginUser) error {
+	has, err := x.Where("external_id=? AND login_source_id=?", externalLoginUser.ExternalID, externalLoginUser.LoginSourceID).
+		NoAutoCondition().
+		Exist(externalLoginUser)
 	if err != nil {
 		return err
 	} else if has {
-		return ErrExternalLoginUserAlreadyExist{gothUser.UserID, user.ID, loginSource.ID}
+		return ErrExternalLoginUserAlreadyExist{externalLoginUser.ExternalID, user.ID, externalLoginUser.LoginSourceID}
 	}
 
 	_, err = x.Insert(externalLoginUser)
@@ -72,3 +85,97 @@ func removeAllAccountLinks(e Engine, user *User) error {
 	_, err := e.Delete(&ExternalLoginUser{UserID: user.ID})
 	return err
 }
+
+// GetUserIDByExternalUserID get user id according to provider and userID
+func GetUserIDByExternalUserID(provider string, userID string) (int64, error) {
+	var id int64
+	_, err := x.Table("external_login_user").
+		Select("user_id").
+		Where("provider=?", provider).
+		And("external_id=?", userID).
+		Get(&id)
+	if err != nil {
+		return 0, err
+	}
+	return id, nil
+}
+
+// UpdateExternalUser updates external user's information
+func UpdateExternalUser(user *User, gothUser goth.User) error {
+	loginSource, err := GetActiveOAuth2LoginSourceByName(gothUser.Provider)
+	if err != nil {
+		return err
+	}
+	externalLoginUser := &ExternalLoginUser{
+		ExternalID:        gothUser.UserID,
+		UserID:            user.ID,
+		LoginSourceID:     loginSource.ID,
+		RawData:           gothUser.RawData,
+		Provider:          gothUser.Provider,
+		Email:             gothUser.Email,
+		Name:              gothUser.Name,
+		FirstName:         gothUser.FirstName,
+		LastName:          gothUser.LastName,
+		NickName:          gothUser.NickName,
+		Description:       gothUser.Description,
+		AvatarURL:         gothUser.AvatarURL,
+		Location:          gothUser.Location,
+		AccessToken:       gothUser.AccessToken,
+		AccessTokenSecret: gothUser.AccessTokenSecret,
+		RefreshToken:      gothUser.RefreshToken,
+		ExpiresAt:         gothUser.ExpiresAt,
+	}
+
+	has, err := x.Where("external_id=? AND login_source_id=?", gothUser.UserID, loginSource.ID).
+		NoAutoCondition().
+		Exist(externalLoginUser)
+	if err != nil {
+		return err
+	} else if !has {
+		return ErrExternalLoginUserNotExist{user.ID, loginSource.ID}
+	}
+
+	_, err = x.Where("external_id=? AND login_source_id=?", gothUser.UserID, loginSource.ID).AllCols().Update(externalLoginUser)
+	return err
+}
+
+// FindExternalUserOptions represents an options to find external users
+type FindExternalUserOptions struct {
+	Provider string
+	Limit    int
+	Start    int
+}
+
+func (opts FindExternalUserOptions) toConds() builder.Cond {
+	var cond = builder.NewCond()
+	if len(opts.Provider) > 0 {
+		cond = cond.And(builder.Eq{"provider": opts.Provider})
+	}
+	return cond
+}
+
+// FindExternalUsersByProvider represents external users via provider
+func FindExternalUsersByProvider(opts FindExternalUserOptions) ([]ExternalLoginUser, error) {
+	var users []ExternalLoginUser
+	err := x.Where(opts.toConds()).
+		Limit(opts.Limit, opts.Start).
+		Asc("id").
+		Find(&users)
+	if err != nil {
+		return nil, err
+	}
+	return users, nil
+}
+
+// UpdateMigrationsByType updates all migrated repositories' posterid from gitServiceType to replace originalAuthorID to posterID
+func UpdateMigrationsByType(tp structs.GitServiceType, externalUserID, userID int64) error {
+	if err := UpdateIssuesMigrationsByType(tp, externalUserID, userID); err != nil {
+		return err
+	}
+
+	if err := UpdateCommentsMigrationsByType(tp, externalUserID, userID); err != nil {
+		return err
+	}
+
+	return UpdateReleasesMigrationsByType(tp, externalUserID, userID)
+}
diff --git a/models/issue.go b/models/issue.go
index 8ce7d496ab..fc675a3ffb 100644
--- a/models/issue.go
+++ b/models/issue.go
@@ -14,6 +14,7 @@ import (
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/structs"
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/util"
@@ -32,7 +33,7 @@ type Issue struct {
 	PosterID         int64       `xorm:"INDEX"`
 	Poster           *User       `xorm:"-"`
 	OriginalAuthor   string
-	OriginalAuthorID int64
+	OriginalAuthorID int64      `xorm:"index"`
 	Title            string     `xorm:"name"`
 	Content          string     `xorm:"TEXT"`
 	RenderedContent  string     `xorm:"-"`
@@ -1947,3 +1948,16 @@ func (issue *Issue) ResolveMentionsByVisibility(ctx DBContext, doer *User, menti
 
 	return
 }
+
+// UpdateIssuesMigrationsByType updates all migrated repositories' issues from gitServiceType to replace originalAuthorID to posterID
+func UpdateIssuesMigrationsByType(gitServiceType structs.GitServiceType, originalAuthorID, posterID int64) error {
+	_, err := x.Table("issue").
+		Where("repo_id IN (SELECT id FROM repository WHERE original_service_type = ?)", gitServiceType).
+		And("original_author_id = ?", originalAuthorID).
+		Update(map[string]interface{}{
+			"poster_id":          posterID,
+			"original_author":    "",
+			"original_author_id": 0,
+		})
+	return err
+}
diff --git a/models/issue_comment.go b/models/issue_comment.go
index 7d38302b98..3a090c3b19 100644
--- a/models/issue_comment.go
+++ b/models/issue_comment.go
@@ -14,6 +14,7 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/markup/markdown"
 	"code.gitea.io/gitea/modules/references"
+	"code.gitea.io/gitea/modules/structs"
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/timeutil"
 
@@ -1022,3 +1023,23 @@ func fetchCodeCommentsByReview(e Engine, issue *Issue, currentUser *User, review
 func FetchCodeComments(issue *Issue, currentUser *User) (CodeComments, error) {
 	return fetchCodeComments(x, issue, currentUser)
 }
+
+// UpdateCommentsMigrationsByType updates comments' migrations information via given git service type and original id and poster id
+func UpdateCommentsMigrationsByType(tp structs.GitServiceType, originalAuthorID, posterID int64) error {
+	_, err := x.Table("comment").
+		Where(builder.In("issue_id",
+			builder.Select("issue.id").
+				From("issue").
+				InnerJoin("repository", "issue.repo_id = repository.id").
+				Where(builder.Eq{
+					"repository.original_service_type": tp,
+				}),
+		)).
+		And("comment.original_author_id = ?", originalAuthorID).
+		Update(map[string]interface{}{
+			"poster_id":          posterID,
+			"original_author":    "",
+			"original_author_id": 0,
+		})
+	return err
+}
diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go
index ef5cd377a6..60a416c6e9 100644
--- a/models/migrations/migrations.go
+++ b/models/migrations/migrations.go
@@ -254,6 +254,8 @@ var migrations = []Migration{
 	NewMigration("add original author name and id on migrated release", addOriginalAuthorOnMigratedReleases),
 	// v99 -> v100
 	NewMigration("add task table and status column for repository table", addTaskTable),
+	// v100 -> v101
+	NewMigration("update migration repositories' service type", updateMigrationServiceTypes),
 }
 
 // Migrate database to current version
diff --git a/models/migrations/v100.go b/models/migrations/v100.go
new file mode 100644
index 0000000000..ac3b73e2ad
--- /dev/null
+++ b/models/migrations/v100.go
@@ -0,0 +1,83 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/go-xorm/xorm"
+)
+
+func updateMigrationServiceTypes(x *xorm.Engine) error {
+	type Repository struct {
+		ID                  int64
+		OriginalServiceType int    `xorm:"index default(0)"`
+		OriginalURL         string `xorm:"VARCHAR(2048)"`
+	}
+
+	if err := x.Sync2(new(Repository)); err != nil {
+		return err
+	}
+
+	var last int
+	const batchSize = 50
+	for {
+		var results = make([]Repository, 0, batchSize)
+		err := x.Where("original_url <> '' AND original_url IS NOT NULL").
+			And("original_service_type = 0 OR original_service_type IS NULL").
+			OrderBy("id").
+			Limit(batchSize, last).
+			Find(&results)
+		if err != nil {
+			return err
+		}
+		if len(results) == 0 {
+			break
+		}
+		last += len(results)
+
+		const PlainGitService = 1 // 1 plain git service
+		const GithubService = 2   // 2 github.com
+
+		for _, res := range results {
+			u, err := url.Parse(res.OriginalURL)
+			if err != nil {
+				return err
+			}
+			var serviceType = PlainGitService
+			if strings.EqualFold(u.Host, "github.com") {
+				serviceType = GithubService
+			}
+			_, err = x.Exec("UPDATE repository SET original_service_type = ? WHERE id = ?", serviceType, res.ID)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	type ExternalLoginUser struct {
+		ExternalID        string                 `xorm:"pk NOT NULL"`
+		UserID            int64                  `xorm:"INDEX NOT NULL"`
+		LoginSourceID     int64                  `xorm:"pk NOT NULL"`
+		RawData           map[string]interface{} `xorm:"TEXT JSON"`
+		Provider          string                 `xorm:"index VARCHAR(25)"`
+		Email             string
+		Name              string
+		FirstName         string
+		LastName          string
+		NickName          string
+		Description       string
+		AvatarURL         string
+		Location          string
+		AccessToken       string
+		AccessTokenSecret string
+		RefreshToken      string
+		ExpiresAt         time.Time
+	}
+
+	return x.Sync2(new(ExternalLoginUser))
+}
diff --git a/models/release.go b/models/release.go
index 243cc2fa3c..03685e0a44 100644
--- a/models/release.go
+++ b/models/release.go
@@ -12,6 +12,7 @@ import (
 
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/structs"
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/timeutil"
 
@@ -366,3 +367,16 @@ func SyncReleasesWithTags(repo *Repository, gitRepo *git.Repository) error {
 	}
 	return nil
 }
+
+// UpdateReleasesMigrationsByType updates all migrated repositories' releases from gitServiceType to replace originalAuthorID to posterID
+func UpdateReleasesMigrationsByType(gitServiceType structs.GitServiceType, originalAuthorID, posterID int64) error {
+	_, err := x.Table("release").
+		Where("repo_id IN (SELECT id FROM repository WHERE original_service_type = ?)", gitServiceType).
+		And("original_author_id = ?", originalAuthorID).
+		Update(map[string]interface{}{
+			"publisher_id":       posterID,
+			"original_author":    "",
+			"original_author_id": 0,
+		})
+	return err
+}
diff --git a/models/repo.go b/models/repo.go
index 23b1c2ef52..aa2cf06f32 100644
--- a/models/repo.go
+++ b/models/repo.go
@@ -32,6 +32,7 @@ import (
 	"code.gitea.io/gitea/modules/options"
 	"code.gitea.io/gitea/modules/process"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/structs"
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/sync"
 	"code.gitea.io/gitea/modules/timeutil"
@@ -137,16 +138,17 @@ const (
 
 // Repository represents a git repository.
 type Repository struct {
-	ID            int64  `xorm:"pk autoincr"`
-	OwnerID       int64  `xorm:"UNIQUE(s) index"`
-	OwnerName     string `xorm:"-"`
-	Owner         *User  `xorm:"-"`
-	LowerName     string `xorm:"UNIQUE(s) INDEX NOT NULL"`
-	Name          string `xorm:"INDEX NOT NULL"`
-	Description   string `xorm:"TEXT"`
-	Website       string `xorm:"VARCHAR(2048)"`
-	OriginalURL   string `xorm:"VARCHAR(2048)"`
-	DefaultBranch string
+	ID                  int64                  `xorm:"pk autoincr"`
+	OwnerID             int64                  `xorm:"UNIQUE(s) index"`
+	OwnerName           string                 `xorm:"-"`
+	Owner               *User                  `xorm:"-"`
+	LowerName           string                 `xorm:"UNIQUE(s) INDEX NOT NULL"`
+	Name                string                 `xorm:"INDEX NOT NULL"`
+	Description         string                 `xorm:"TEXT"`
+	Website             string                 `xorm:"VARCHAR(2048)"`
+	OriginalServiceType structs.GitServiceType `xorm:"index"`
+	OriginalURL         string                 `xorm:"VARCHAR(2048)"`
+	DefaultBranch       string
 
 	NumWatches          int
 	NumStars            int