Add a new table issue_index to store the max issue index so that issue could be deleted with no duplicated index (#15599)

* Add a new table issue_index to store the max issue index so that issue could be deleted with no duplicated index

* Fix pull index

* Add tests for concurrent creating issues

* Fix lint

* Fix tests

* Fix postgres test

* Add test for migration v180

* Rename wrong test file name

Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: Lauris BH <lauris@nix.lv>

3 files changed, 103 insertions, 0 deletions

diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go
index df1bac4a13..4c07db0a0f 100644
--- a/models/migrations/migrations.go
+++ b/models/migrations/migrations.go
@@ -313,6 +313,8 @@ var migrations = []Migration{
 	NewMigration("Delete credentials from past migrations", deleteMigrationCredentials),
 	// v181 -> v182
 	NewMigration("Always save primary email on email address table", addPrimaryEmail2EmailAddress),
+	// v182 -> v183
+	NewMigration("Add issue resource index table", addIssueResourceIndexTable),
 }
 
 // GetCurrentDBVersion returns the current db version
diff --git a/models/migrations/v182.go b/models/migrations/v182.go
new file mode 100644
index 0000000000..dd9a04f27e
--- /dev/null
+++ b/models/migrations/v182.go
@@ -0,0 +1,42 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"xorm.io/xorm"
+)
+
+func addIssueResourceIndexTable(x *xorm.Engine) error {
+	type ResourceIndex struct {
+		GroupID  int64 `xorm:"index unique(s)"`
+		MaxIndex int64 `xorm:"index unique(s)"`
+	}
+
+	sess := x.NewSession()
+	defer sess.Close()
+
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+
+	if err := sess.Table("issue_index").Sync2(new(ResourceIndex)); err != nil {
+		return err
+	}
+
+	// Remove data we're goint to rebuild
+	if _, err := sess.Table("issue_index").Where("1=1").Delete(&ResourceIndex{}); err != nil {
+		return err
+	}
+
+	// Create current data for all repositories with issues and PRs
+	if _, err := sess.Exec("INSERT INTO issue_index (group_id, max_index) " +
+		"SELECT max_data.repo_id, max_data.max_index " +
+		"FROM ( SELECT issue.repo_id AS repo_id, max(issue.`index`) AS max_index " +
+		"FROM issue GROUP BY issue.repo_id) AS max_data"); err != nil {
+		return err
+	}
+
+	return sess.Commit()
+}
diff --git a/models/migrations/v182_test.go b/models/migrations/v182_test.go
new file mode 100644
index 0000000000..6f418f7794
--- /dev/null
+++ b/models/migrations/v182_test.go
@@ -0,0 +1,59 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_addIssueResourceIndexTable(t *testing.T) {
+	// Create the models used in the migration
+	type Issue struct {
+		ID     int64 `xorm:"pk autoincr"`
+		RepoID int64 `xorm:"UNIQUE(s)"`
+		Index  int64 `xorm:"UNIQUE(s)"`
+	}
+
+	// Prepare and load the testing database
+	x, deferable := prepareTestEnv(t, 0, new(Issue))
+	if x == nil || t.Failed() {
+		defer deferable()
+		return
+	}
+	defer deferable()
+
+	// Run the migration
+	if err := addIssueResourceIndexTable(x); err != nil {
+		assert.NoError(t, err)
+		return
+	}
+
+	type ResourceIndex struct {
+		GroupID  int64 `xorm:"index unique(s)"`
+		MaxIndex int64 `xorm:"index unique(s)"`
+	}
+
+	var start = 0
+	const batchSize = 1000
+	for {
+		var indexes = make([]ResourceIndex, 0, batchSize)
+		err := x.Table("issue_index").Limit(batchSize, start).Find(&indexes)
+		assert.NoError(t, err)
+
+		for _, idx := range indexes {
+			var maxIndex int
+			has, err := x.SQL("SELECT max(`index`) FROM issue WHERE repo_id = ?", idx.GroupID).Get(&maxIndex)
+			assert.NoError(t, err)
+			assert.True(t, has)
+			assert.EqualValues(t, maxIndex, idx.MaxIndex)
+		}
+		if len(indexes) < batchSize {
+			break
+		}
+		start += len(indexes)
+	}
+}