Add support for migrating from Gitlab (#9084)

* First stab at a Gitlab migrations interface.

* Modify JS to show migration for Gitlab

* Properly strip out #gitlab tag from repo name

* Working Gitlab migrations!
Still need to figure out how to hide tokens/etc from showing up in opts.CloneAddr

* Try #2 at trying to hide credentials.
CloneAddr was being used as OriginalURL.
Now passing OriginalURL through from the form and saving it.

* Add go-gitlab dependency

* Vendor go-gitlab

* Use gitlab.BasicAuthClient
Correct CloneURL.
This should be functioning!
Previous commits fixed "Migrated from"
from including the migration credentials.

* Replaced repoPath with repoID globally.
RepoID is grabbed in NewGitlabDownloader

* Logging touchup

* Properly set private repo status.
Properly set milestone deadline time.
Consistently use Gitlab username for 'Name'.

* Add go-gitlab vendor cache

* Fix PR migrations:
- Count of issues is kept to set a non-conflicting PR.ID
- Bool is used to tell whether to fetch Issue or PR comments

* Ensure merged PRs are closed and set with the proper time

* Remove copyright and some commented code

* Rip out '#gitlab' based self-hosted Gitlab support

* Hide given credentials for migrated repos.

CloneAddr was being saved as OriginalURL.

Now passing OriginalURL through from the form and
saving it in it's place

* Use asset.URL directly, no point in parsing.
Opened PRs should fall through to false.

* Fix importing Milestones.
Allow importing using Personal Tokens or anonymous access.

* Fix Gitlab Milestone migration if DueDate isn't set

* Empty Milestone due dates properly return nil, not zero time

* Add GITLAB_READ_TOKEN to drone unit-test step

* Add working gitlab_test.go.
A Personal Access Token, given in env variable GITLAB_READ_TOKEN
is required to run the test.

* Fix linting issues

* Add modified JS files

* Remove pre-build JS files

* Only merged PRs are marged as merged/closed

* Test topics

* Skip test if gitlab is inaccessible

* Grab personal token from username, not password.
Matches Github migration implementation

* Add SetContext() to GitlabDownloader.

* Checking Updated field in Issues.

* Actually fetch Issue Updated time from Gitlab

* Add Gitlab migration GetReviews() stub

* Fix Patch and Clone URLs

* check Updated too

* fix mod

* make vendor with go1.14

Co-authored-by: techknowlogick <techknowlogick@gitea.io>
Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: Lauris BH <lauris@nix.lv>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>

1 files changed, 246 insertions, 0 deletions

diff --git a/modules/migrations/gitlab_test.go b/modules/migrations/gitlab_test.go
new file mode 100644
index 0000000000..8525d7e9ce
--- /dev/null
+++ b/modules/migrations/gitlab_test.go
@@ -0,0 +1,246 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"net/http"
+	"os"
+	"testing"
+	"time"
+
+	"code.gitea.io/gitea/modules/migrations/base"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGitlabDownloadRepo(t *testing.T) {
+	// Skip tests if Gitlab token is not found
+	gitlabPersonalAccessToken := os.Getenv("GITLAB_READ_TOKEN")
+	if gitlabPersonalAccessToken == "" {
+		t.Skip("skipped test because GITLAB_READ_TOKEN was not in the environment")
+	}
+
+	resp, err := http.Get("https://gitlab.com/gitea/test_repo")
+	if err != nil || resp.StatusCode != 200 {
+		t.Skipf("Can't access test repo, skipping %s", t.Name())
+	}
+
+	downloader := NewGitlabDownloader("https://gitlab.com", "gitea/test_repo", gitlabPersonalAccessToken, "")
+	if downloader == nil {
+		t.Fatal("NewGitlabDownloader is nil")
+	}
+	repo, err := downloader.GetRepoInfo()
+	assert.NoError(t, err)
+	// Repo Owner is blank in Gitlab Group repos
+	assert.EqualValues(t, &base.Repository{
+		Name:        "test_repo",
+		Owner:       "",
+		Description: "Test repository for testing migration from gitlab to gitea",
+		CloneURL:    "https://gitlab.com/gitea/test_repo.git",
+		OriginalURL: "https://gitlab.com/gitea/test_repo",
+	}, repo)
+
+	topics, err := downloader.GetTopics()
+	assert.NoError(t, err)
+	assert.True(t, len(topics) == 2)
+	assert.EqualValues(t, []string{"migration", "test"}, topics)
+
+	milestones, err := downloader.GetMilestones()
+	assert.NoError(t, err)
+	assert.True(t, len(milestones) >= 2)
+
+	for _, milestone := range milestones {
+		switch milestone.Title {
+		case "1.0":
+			assertMilestoneEqual(t, "", "1.0",
+				"",
+				"2019-11-28 08:42:30.301 +0000 UTC",
+				"2019-11-28 15:57:52.401 +0000 UTC",
+				"",
+				"closed", milestone)
+		case "1.1.0":
+			assertMilestoneEqual(t, "", "1.1.0",
+				"",
+				"2019-11-28 08:42:44.575 +0000 UTC",
+				"2019-11-28 08:42:44.575 +0000 UTC",
+				"",
+				"active", milestone)
+		}
+	}
+
+	labels, err := downloader.GetLabels()
+	assert.NoError(t, err)
+	assert.True(t, len(labels) >= 9)
+	for _, l := range labels {
+		switch l.Name {
+		case "bug":
+			assertLabelEqual(t, "bug", "d9534f", "", l)
+		case "documentation":
+			assertLabelEqual(t, "documentation", "f0ad4e", "", l)
+		case "confirmed":
+			assertLabelEqual(t, "confirmed", "d9534f", "", l)
+		case "enhancement":
+			assertLabelEqual(t, "enhancement", "5cb85c", "", l)
+		case "critical":
+			assertLabelEqual(t, "critical", "d9534f", "", l)
+		case "discussion":
+			assertLabelEqual(t, "discussion", "428bca", "", l)
+		case "suggestion":
+			assertLabelEqual(t, "suggestion", "428bca", "", l)
+		case "support":
+			assertLabelEqual(t, "support", "f0ad4e", "", l)
+		case "duplicate":
+			assertLabelEqual(t, "duplicate", "7F8C8D", "", l)
+		}
+	}
+
+	releases, err := downloader.GetReleases()
+	assert.NoError(t, err)
+	assert.EqualValues(t, []*base.Release{
+		{
+			TagName:         "v0.9.99",
+			TargetCommitish: "0720a3ec57c1f843568298117b874319e7deee75",
+			Name:            "First Release",
+			Body:            "A test release",
+			Created:         time.Date(2019, 11, 28, 9, 9, 48, 840000000, time.UTC),
+			PublisherID:     1241334,
+			PublisherName:   "lafriks",
+		},
+	}, releases[len(releases)-1:])
+
+	// downloader.GetIssues()
+	issues, isEnd, err := downloader.GetIssues(1, 2)
+	assert.NoError(t, err)
+	assert.EqualValues(t, 2, len(issues))
+	assert.False(t, isEnd)
+
+	var (
+		closed1 = time.Date(2019, 11, 28, 8, 46, 23, 275000000, time.UTC)
+		closed2 = time.Date(2019, 11, 28, 8, 45, 44, 959000000, time.UTC)
+	)
+	assert.EqualValues(t, []*base.Issue{
+		{
+			Number:     1,
+			Title:      "Please add an animated gif icon to the merge button",
+			Content:    "I just want the merge button to hurt my eyes a little. :stuck_out_tongue_closed_eyes:",
+			Milestone:  "1.0.0",
+			PosterID:   1241334,
+			PosterName: "lafriks",
+			State:      "closed",
+			Created:    time.Date(2019, 11, 28, 8, 43, 35, 459000000, time.UTC),
+			Updated:    time.Date(2019, 11, 28, 8, 46, 23, 275000000, time.UTC),
+			Labels: []*base.Label{
+				{
+					Name: "bug",
+				},
+				{
+					Name: "discussion",
+				},
+			},
+			Reactions: nil,
+			Closed:    &closed1,
+		},
+		{
+			Number:     2,
+			Title:      "Test issue",
+			Content:    "This is test issue 2, do not touch!",
+			Milestone:  "1.1.0",
+			PosterID:   1241334,
+			PosterName: "lafriks",
+			State:      "closed",
+			Created:    time.Date(2019, 11, 28, 8, 44, 46, 277000000, time.UTC),
+			Updated:    time.Date(2019, 11, 28, 8, 45, 44, 987000000, time.UTC),
+			Labels: []*base.Label{
+				{
+					Name: "duplicate",
+				},
+			},
+			Reactions: nil,
+			Closed:    &closed2,
+		},
+	}, issues)
+
+	// downloader.GetComments()
+	comments, err := downloader.GetComments(2)
+	assert.NoError(t, err)
+	assert.EqualValues(t, 4, len(comments))
+	assert.EqualValues(t, []*base.Comment{
+		{
+			IssueIndex: 2,
+			PosterID:   1241334,
+			PosterName: "lafriks",
+			Created:    time.Date(2019, 11, 28, 8, 44, 52, 501000000, time.UTC),
+			Updated:    time.Date(2019, 11, 28, 8, 44, 52, 501000000, time.UTC),
+			Content:    "This is a comment",
+			Reactions:  nil,
+		},
+		{
+			IssueIndex: 2,
+			PosterID:   1241334,
+			PosterName: "lafriks",
+			Created:    time.Date(2019, 11, 28, 8, 45, 2, 329000000, time.UTC),
+			Content:    "changed milestone to %2",
+			Reactions:  nil,
+		},
+		{
+			IssueIndex: 2,
+			PosterID:   1241334,
+			PosterName: "lafriks",
+			Created:    time.Date(2019, 11, 28, 8, 45, 45, 7000000, time.UTC),
+			Content:    "closed",
+			Reactions:  nil,
+		},
+		{
+			IssueIndex: 2,
+			PosterID:   1241334,
+			PosterName: "lafriks",
+			Created:    time.Date(2019, 11, 28, 8, 45, 53, 501000000, time.UTC),
+			Content:    "A second comment",
+			Reactions:  nil,
+		},
+	}, comments[:4])
+
+	// downloader.GetPullRequests()
+	prs, err := downloader.GetPullRequests(1, 1)
+	assert.NoError(t, err)
+	assert.EqualValues(t, 1, len(prs))
+
+	assert.EqualValues(t, []*base.PullRequest{
+		{
+			Number:     4,
+			Title:      "Test branch",
+			Content:    "do not merge this PR",
+			Milestone:  "1.0.0",
+			PosterID:   1241334,
+			PosterName: "lafriks",
+			State:      "opened",
+			Created:    time.Date(2019, 11, 28, 15, 56, 54, 104000000, time.UTC),
+			Updated:    time.Date(2019, 11, 28, 15, 56, 54, 104000000, time.UTC),
+			Labels: []*base.Label{
+				{
+					Name: "bug",
+				},
+			},
+			PatchURL: "https://gitlab.com/gitea/test_repo/-/merge_requests/2.patch",
+			Head: base.PullRequestBranch{
+				Ref:       "feat/test",
+				CloneURL:  "https://gitlab.com/gitea/test_repo/-/merge_requests/2",
+				SHA:       "9f733b96b98a4175276edf6a2e1231489c3bdd23",
+				RepoName:  "test_repo",
+				OwnerName: "lafriks",
+			},
+			Base: base.PullRequestBranch{
+				Ref:       "master",
+				SHA:       "",
+				OwnerName: "lafriks",
+				RepoName:  "test_repo",
+			},
+			Closed:         nil,
+			Merged:         false,
+			MergedTime:     nil,
+			MergeCommitSHA: "",
+		},
+	}, prs)
+}