LINUX_ARCHS ?= linux/amd64,linux/386,linux/arm-5,linux/arm-6,linux/arm64
-GO_PACKAGES ?= $(filter-out code.gitea.io/gitea/integrations/migration-test,$(filter-out code.gitea.io/gitea/integrations,$(shell $(GO) list -mod=vendor ./... | grep -v /vendor/)))
+GO_PACKAGES ?= $(filter-out code.gitea.io/gitea/models/migrations code.gitea.io/gitea/integrations/migration-test code.gitea.io/gitea/integrations,$(shell $(GO) list -mod=vendor ./... | grep -v /vendor/))
FOMANTIC_CONFIGS := semantic.json web_src/fomantic/theme.config.less web_src/fomantic/_site/globals/site.variables
FOMANTIC_DEST := web_src/fomantic/build/semantic.js web_src/fomantic/build/semantic.css
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/sqlite.ini ./integrations.sqlite.test -test.run $(subst .,/,$*)
.PHONY: test-sqlite-migration
-test-sqlite-migration: migrations.sqlite.test generate-ini-sqlite
+test-sqlite-migration: migrations.sqlite.test migrations.individual.sqlite.test generate-ini-sqlite
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/sqlite.ini ./migrations.sqlite.test
+ GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/sqlite.ini ./migrations.individual.sqlite.test
generate-ini-mysql:
sed -e 's|{{TEST_MYSQL_HOST}}|${TEST_MYSQL_HOST}|g' \
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mysql.ini ./integrations.mysql.test -test.run $(subst .,/,$*)
.PHONY: test-mysql-migration
-test-mysql-migration: migrations.mysql.test generate-ini-mysql
+test-mysql-migration: migrations.mysql.test migrations.individual.mysql.test generate-ini-mysql
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mysql.ini ./migrations.mysql.test
+ GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mysql.ini ./migrations.individual.mysql.test
generate-ini-mysql8:
sed -e 's|{{TEST_MYSQL8_HOST}}|${TEST_MYSQL8_HOST}|g' \
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mysql8.ini ./integrations.mysql8.test -test.run $(subst .,/,$*)
.PHONY: test-mysql8-migration
-test-mysql8-migration: migrations.mysql8.test generate-ini-mysql8
+test-mysql8-migration: migrations.mysql8.test migrations.individual.mysql8.test generate-ini-mysql8
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mysql8.ini ./migrations.mysql8.test
+ GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mysql8.ini ./migrations.individual.mysql8.test
generate-ini-pgsql:
sed -e 's|{{TEST_PGSQL_HOST}}|${TEST_PGSQL_HOST}|g' \
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/pgsql.ini ./integrations.pgsql.test -test.run $(subst .,/,$*)
.PHONY: test-pgsql-migration
-test-pgsql-migration: migrations.pgsql.test generate-ini-pgsql
+test-pgsql-migration: migrations.pgsql.test migrations.individual.pgsql.test generate-ini-pgsql
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/pgsql.ini ./migrations.pgsql.test
+ GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/pgsql.ini ./migrations.individual.pgsql.test
generate-ini-mssql:
sed -e 's|{{TEST_MSSQL_HOST}}|${TEST_MSSQL_HOST}|g' \
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mssql.ini ./integrations.mssql.test -test.run $(subst .,/,$*)
.PHONY: test-mssql-migration
-test-mssql-migration: migrations.mssql.test generate-ini-mssql
+test-mssql-migration: migrations.mssql.test migrations.individual.mssql.test generate-ini-mssql
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mssql.ini ./migrations.mssql.test -test.failfast
+ GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mssql.ini ./migrations.individual.mssql.test -test.failfast
.PHONY: bench-sqlite
bench-sqlite: integrations.sqlite.test generate-ini-sqlite
migrations.sqlite.test: $(GO_SOURCES)
$(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/integrations/migration-test -o migrations.sqlite.test -tags '$(TEST_TAGS)'
+.PHONY: migrations.individual.mysql.test
+migrations.individual.mysql.test: $(GO_SOURCES)
+ $(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/models/migrations -o migrations.individual.mysql.test
+
+.PHONY: migrations.individual.mysql8.test
+migrations.individual.mysql8.test: $(GO_SOURCES)
+ $(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/models/migrations -o migrations.individual.mysql8.test
+
+.PHONY: migrations.individual.pgsql.test
+migrations.individual.pgsql.test: $(GO_SOURCES)
+ $(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/models/migrations -o migrations.individual.pgsql.test
+
+.PHONY: migrations.individual.mssql.test
+migrations.individual.mssql.test: $(GO_SOURCES)
+ $(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/models/migrations -o migrations.individual.mssql.test
+
+.PHONY: migrations.individual.sqlite.test
+migrations.individual.sqlite.test: $(GO_SOURCES)
+ $(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/models/migrations -o migrations.individual.sqlite.test -tags '$(TEST_TAGS)'
+
.PHONY: check
check: test
--- /dev/null
+# Issue_Label 1 should not be deleted
+-
+ id: 1
+ issue_id: 1
+ label_id: 1
+
+# Issue_label 2 should be deleted
+-
+ id: 2
+ issue_id: 5
+ label_id: 99
+
+# Issue_Label 3 should not be deleted
+-
+ id: 3
+ issue_id: 2
+ label_id: 1
+
+# Issue_Label 4 should not be deleted
+-
+ id: 4
+ issue_id: 2
+ label_id: 4
+
+-
+ id: 5
+ issue_id: 2
+ label_id: 87
+
--- /dev/null
+-
+ id: 1
+ repo_id: 1
+ org_id: 0
+ name: label1
+ color: '#abcdef'
+ num_issues: 2
+ num_closed_issues: 0
+
+-
+ id: 2
+ repo_id: 1
+ org_id: 0
+ name: label2
+ color: '#000000'
+ num_issues: 1
+ num_closed_issues: 1
+-
+ id: 3
+ repo_id: 0
+ org_id: 3
+ name: orglabel3
+ color: '#abcdef'
+ num_issues: 0
+ num_closed_issues: 0
+
+-
+ id: 4
+ repo_id: 0
+ org_id: 3
+ name: orglabel4
+ color: '#000000'
+ num_issues: 1
+ num_closed_issues: 0
+
+-
+ id: 5
+ repo_id: 10
+ org_id: 0
+ name: pull-test-label
+ color: '#000000'
+ num_issues: 0
+ num_closed_issues: 0
--- /dev/null
+# type Comment struct {
+# ID int64 `xorm:"pk autoincr"`
+# Type int `xorm:"INDEX"`
+# IssueID int64 `xorm:"INDEX"`
+# LabelID int64
+# }
+#
+# we are only interested in type 7
+#
+
+-
+ id: 1 # Should remain
+ type: 6
+ issue_id: 1
+ label_id: 0
+ should_remain: true
+-
+ id: 2 # Should remain
+ type: 7
+ issue_id: 1 # repo_id: 1
+ label_id: 1 # repo_id: 1
+ should_remain: true
+-
+ id: 3 # Should remain
+ type: 7
+ issue_id: 2 # repo_id: 2 owner_id: 1
+ label_id: 2 # org_id: 1
+ should_remain: true
+-
+ id: 4 # Should be DELETED
+ type: 7
+ issue_id: 1 # repo_id: 1
+ label_id: 3 # repo_id: 2
+ should_remain: false
+-
+ id: 5 # Should remain
+ type: 7
+ issue_id: 3 # repo_id: 1
+ label_id: 1 # repo_id: 1
+ should_remain: true
+-
+ id: 6 # Should be DELETED
+ type: 7
+ issue_id: 3 # repo_id: 1 owner_id: 2
+ label_id: 2 # org_id: 1
+ should_remain: false
+-
+ id: 7 # Should be DELETED
+ type: 7
+ issue_id: 3 # repo_id: 1 owner_id: 2
+ label_id: 5 # repo_id: 3
+ should_remain: false
--- /dev/null
+# type Issue struct {
+# ID int64 `xorm:"pk autoincr"`
+# RepoID int64 `xorm:"INDEX UNIQUE(repo_index)"`
+# Index int64 `xorm:"UNIQUE(repo_index)"` // Index in one repository.
+# }
+-
+ id: 1
+ repo_id: 1
+ index: 1
+-
+ id: 2
+ repo_id: 2
+ index: 1
+-
+ id: 3
+ repo_id: 1
+ index: 2
+-
+ id: 4
+ repo_id: 3
+ index: 1
--- /dev/null
+# type IssueLabel struct {
+# ID int64 `xorm:"pk autoincr"`
+# IssueID int64 `xorm:"UNIQUE(s)"`
+# LabelID int64 `xorm:"UNIQUE(s)"`
+# }
+-
+ id: 1 # Should remain - matches comment 2
+ issue_id: 1
+ label_id: 1
+ should_remain: true
+-
+ id: 2 # Should remain
+ issue_id: 2
+ label_id: 2
+ should_remain: true
+-
+ id: 3 # Should be deleted
+ issue_id: 1
+ label_id: 3
+ should_remain: false
+-
+ id: 4 # Should remain
+ issue_id: 3
+ label_id: 1
+ should_remain: true
+-
+ id: 5 # Should be deleted
+ issue_id: 3
+ label_id: 2
+ should_remain: false
+-
+ id: 6 # Should be deleted
+ issue_id: 3
+ label_id: 5
+ should_remain: false
--- /dev/null
+# type Label struct {
+# ID int64 `xorm:"pk autoincr"`
+# RepoID int64 `xorm:"INDEX"`
+# OrgID int64 `xorm:"INDEX"`
+# }
+-
+ id: 1
+ repo_id: 1
+ org_id: 0
+-
+ id: 2
+ repo_id: 0
+ org_id: 1
+-
+ id: 3
+ repo_id: 2
+ org_id: 0
+-
+ id: 4
+ repo_id: 1
+ org_id: 0
+-
+ id: 5
+ repo_id: 3
+ org_id: 0
+
--- /dev/null
+# type Repository struct {
+# ID int64 `xorm:"pk autoincr"`
+# OwnerID int64 `xorm:"UNIQUE(s) index"`
+# LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"`
+# }
+-
+ id: 1
+ owner_id: 2
+ lower_name: "repo1"
+-
+ id: 2
+ owner_id: 1
+ lower_name: "repo2"
+-
+ id: 3
+ owner_id: 2
+ lower_name: "repo3"
}
for _, constraint := range constraints {
if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` DROP CONSTRAINT `%s`", tableName, constraint)); err != nil {
- return fmt.Errorf("Drop table `%s` constraint `%s`: %v", tableName, constraint, err)
+ return fmt.Errorf("Drop table `%s` default constraint `%s`: %v", tableName, constraint, err)
}
}
+ sql = fmt.Sprintf("SELECT DISTINCT Name FROM SYS.INDEXES INNER JOIN SYS.INDEX_COLUMNS ON INDEXES.INDEX_ID = INDEX_COLUMNS.INDEX_ID AND INDEXES.OBJECT_ID = INDEX_COLUMNS.OBJECT_ID WHERE INDEXES.OBJECT_ID = OBJECT_ID('%[1]s') AND INDEX_COLUMNS.COLUMN_ID IN (SELECT column_id FROM sys.columns WHERE lower(NAME) IN (%[2]s) AND object_id = OBJECT_ID('%[1]s'))",
+ tableName, strings.ReplaceAll(cols, "`", "'"))
+ constraints = make([]string, 0)
+ if err := sess.SQL(sql).Find(&constraints); err != nil {
+ return fmt.Errorf("Find constraints: %v", err)
+ }
+ for _, constraint := range constraints {
+ if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` DROP CONSTRAINT IF EXISTS `%s`", tableName, constraint)); err != nil {
+ return fmt.Errorf("Drop table `%s` index constraint `%s`: %v", tableName, constraint, err)
+ }
+ if _, err := sess.Exec(fmt.Sprintf("DROP INDEX IF EXISTS `%[2]s` ON `%[1]s`", tableName, constraint)); err != nil {
+ return fmt.Errorf("Drop index `%[2]s` on `%[1]s`: %v", tableName, constraint, err)
+ }
+ }
+
if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` DROP COLUMN %s", tableName, cols)); err != nil {
return fmt.Errorf("Drop table `%s` columns %v: %v", tableName, columnNames, err)
}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "database/sql"
+ "fmt"
+ "os"
+ "path"
+ "path/filepath"
+ "runtime"
+ "testing"
+ "time"
+
+ "code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/modules/base"
+ "code.gitea.io/gitea/modules/setting"
+ "code.gitea.io/gitea/modules/timeutil"
+ "code.gitea.io/gitea/modules/util"
+ "github.com/stretchr/testify/assert"
+ "github.com/unknwon/com"
+ "xorm.io/xorm"
+ "xorm.io/xorm/names"
+)
+
+func TestMain(m *testing.M) {
+ giteaRoot := base.SetupGiteaRoot()
+ if giteaRoot == "" {
+ fmt.Println("Environment variable $GITEA_ROOT not set")
+ os.Exit(1)
+ }
+ giteaBinary := "gitea"
+ if runtime.GOOS == "windows" {
+ giteaBinary += ".exe"
+ }
+ setting.AppPath = path.Join(giteaRoot, giteaBinary)
+ if _, err := os.Stat(setting.AppPath); err != nil {
+ fmt.Printf("Could not find gitea binary at %s\n", setting.AppPath)
+ os.Exit(1)
+ }
+
+ giteaConf := os.Getenv("GITEA_CONF")
+ if giteaConf == "" {
+ giteaConf = path.Join(filepath.Dir(setting.AppPath), "integrations/sqlite.ini")
+ fmt.Printf("Environment variable $GITEA_CONF not set - defaulting to %s\n", giteaConf)
+ }
+
+ if !path.IsAbs(giteaConf) {
+ setting.CustomConf = path.Join(giteaRoot, giteaConf)
+ } else {
+ setting.CustomConf = giteaConf
+ }
+
+ setting.SetCustomPathAndConf("", "", "")
+ setting.NewContext()
+ setting.CheckLFSVersion()
+ setting.InitDBConfig()
+ setting.NewLogServices(true)
+
+ exitStatus := m.Run()
+
+ if err := removeAllWithRetry(setting.RepoRootPath); err != nil {
+ fmt.Fprintf(os.Stderr, "os.RemoveAll: %v\n", err)
+ }
+ if err := removeAllWithRetry(setting.AppDataPath); err != nil {
+ fmt.Fprintf(os.Stderr, "os.RemoveAll: %v\n", err)
+ }
+ os.Exit(exitStatus)
+}
+
+func removeAllWithRetry(dir string) error {
+ var err error
+ for i := 0; i < 20; i++ {
+ err = os.RemoveAll(dir)
+ if err == nil {
+ break
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+ return err
+}
+
+// SetEngine sets the xorm.Engine
+func SetEngine() (*xorm.Engine, error) {
+ x, err := models.GetNewEngine()
+ if err != nil {
+ return x, fmt.Errorf("Failed to connect to database: %v", err)
+ }
+
+ x.SetMapper(names.GonicMapper{})
+ // WARNING: for serv command, MUST remove the output to os.stdout,
+ // so use log file to instead print to stdout.
+ x.SetLogger(models.NewXORMLogger(setting.Database.LogSQL))
+ x.ShowSQL(setting.Database.LogSQL)
+ x.SetMaxOpenConns(setting.Database.MaxOpenConns)
+ x.SetMaxIdleConns(setting.Database.MaxIdleConns)
+ x.SetConnMaxLifetime(setting.Database.ConnMaxLifetime)
+ return x, nil
+}
+
+func deleteDB() error {
+ switch {
+ case setting.Database.UseSQLite3:
+ if err := util.Remove(setting.Database.Path); err != nil {
+ return err
+ }
+ return os.MkdirAll(path.Dir(setting.Database.Path), os.ModePerm)
+
+ case setting.Database.UseMySQL:
+ db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/",
+ setting.Database.User, setting.Database.Passwd, setting.Database.Host))
+ if err != nil {
+ return err
+ }
+ defer db.Close()
+
+ if _, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", setting.Database.Name)); err != nil {
+ return err
+ }
+
+ if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", setting.Database.Name)); err != nil {
+ return err
+ }
+ return nil
+ case setting.Database.UsePostgreSQL:
+ db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s",
+ setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.SSLMode))
+ if err != nil {
+ return err
+ }
+ defer db.Close()
+
+ if _, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", setting.Database.Name)); err != nil {
+ return err
+ }
+
+ if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", setting.Database.Name)); err != nil {
+ return err
+ }
+ db.Close()
+
+ // Check if we need to setup a specific schema
+ if len(setting.Database.Schema) != 0 {
+ db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s",
+ setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name, setting.Database.SSLMode))
+ if err != nil {
+ return err
+ }
+ defer db.Close()
+
+ schrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM information_schema.schemata WHERE schema_name = '%s'", setting.Database.Schema))
+ if err != nil {
+ return err
+ }
+ defer schrows.Close()
+
+ if !schrows.Next() {
+ // Create and setup a DB schema
+ _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", setting.Database.Schema))
+ if err != nil {
+ return err
+ }
+ }
+
+ // Make the user's default search path the created schema; this will affect new connections
+ _, err = db.Exec(fmt.Sprintf(`ALTER USER "%s" SET search_path = %s`, setting.Database.User, setting.Database.Schema))
+ if err != nil {
+ return err
+ }
+ return nil
+ }
+ case setting.Database.UseMSSQL:
+ host, port := setting.ParseMSSQLHostPort(setting.Database.Host)
+ db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;",
+ host, port, "master", setting.Database.User, setting.Database.Passwd))
+ if err != nil {
+ return err
+ }
+ defer db.Close()
+
+ if _, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS [%s]", setting.Database.Name)); err != nil {
+ return err
+ }
+ if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE [%s]", setting.Database.Name)); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+// prepareTestEnv prepares the test environment and reset the database. The skip parameter should usually be 0.
+// Provide models to be sync'd with the database - in particular any models you expect fixtures to be loaded from.
+//
+// fixtures in `models/migrations/fixtures/<TestName>` will be loaded automatically
+func prepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.Engine, func()) {
+ t.Helper()
+ ourSkip := 2
+ ourSkip += skip
+ deferFn := PrintCurrentTest(t, ourSkip)
+ assert.NoError(t, os.RemoveAll(setting.RepoRootPath))
+
+ assert.NoError(t, com.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"),
+ setting.RepoRootPath))
+
+ if err := deleteDB(); err != nil {
+ t.Errorf("unable to reset database: %v", err)
+ return nil, deferFn
+ }
+
+ x, err := SetEngine()
+ assert.NoError(t, err)
+ if x != nil {
+ oldDefer := deferFn
+ deferFn = func() {
+ oldDefer()
+ if err := x.Close(); err != nil {
+ t.Errorf("error during close: %v", err)
+ }
+ }
+ }
+ if err != nil {
+ return x, deferFn
+ }
+
+ if len(syncModels) > 0 {
+ if err := x.Sync2(syncModels...); err != nil {
+ t.Errorf("error during sync: %v", err)
+ return x, deferFn
+ }
+ }
+
+ fixturesDir := filepath.Join(filepath.Dir(setting.AppPath), "models", "migrations", "fixtures", t.Name())
+
+ if _, err := os.Stat(fixturesDir); err == nil {
+ t.Logf("initializing fixtures from: %s", fixturesDir)
+ if err := models.InitFixtures(fixturesDir, x); err != nil {
+ t.Errorf("error whilst initializing fixtures from %s: %v", fixturesDir, err)
+ return x, deferFn
+ }
+ if err := models.LoadFixtures(x); err != nil {
+ t.Errorf("error whilst loading fixtures from %s: %v", fixturesDir, err)
+ return x, deferFn
+ }
+ } else if !os.IsNotExist(err) {
+ t.Errorf("unexpected error whilst checking for existence of fixtures: %v", err)
+ } else {
+ t.Logf("no fixtures found in: %s", fixturesDir)
+ }
+
+ return x, deferFn
+}
+
+func Test_dropTableColumns(t *testing.T) {
+ x, deferable := prepareTestEnv(t, 0)
+ if x == nil || t.Failed() {
+ defer deferable()
+ return
+ }
+ defer deferable()
+
+ type DropTest struct {
+ ID int64 `xorm:"pk autoincr"`
+ FirstColumn string
+ ToDropColumn string `xorm:"unique"`
+ AnotherColumn int64
+ CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
+ UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
+ }
+
+ columns := []string{
+ "first_column",
+ "to_drop_column",
+ "another_column",
+ "created_unix",
+ "updated_unix",
+ }
+
+ for i := range columns {
+ x.SetMapper(names.GonicMapper{})
+ if err := x.Sync2(new(DropTest)); err != nil {
+ t.Errorf("unable to create DropTest table: %v", err)
+ return
+ }
+ sess := x.NewSession()
+ if err := sess.Begin(); err != nil {
+ sess.Close()
+ t.Errorf("unable to begin transaction: %v", err)
+ return
+ }
+ if err := dropTableColumns(sess, "drop_test", columns[i:]...); err != nil {
+ sess.Close()
+ t.Errorf("Unable to drop columns[%d:]: %s from drop_test: %v", i, columns[i:], err)
+ return
+ }
+ if err := sess.Commit(); err != nil {
+ sess.Close()
+ t.Errorf("unable to commit transaction: %v", err)
+ return
+ }
+ sess.Close()
+ if err := x.DropTables(new(DropTest)); err != nil {
+ t.Errorf("unable to drop table: %v", err)
+ return
+ }
+ for j := range columns[i+1:] {
+ x.SetMapper(names.GonicMapper{})
+ if err := x.Sync2(new(DropTest)); err != nil {
+ t.Errorf("unable to create DropTest table: %v", err)
+ return
+ }
+ dropcols := append([]string{columns[i]}, columns[j+i+1:]...)
+ sess := x.NewSession()
+ if err := sess.Begin(); err != nil {
+ sess.Close()
+ t.Errorf("unable to begin transaction: %v", err)
+ return
+ }
+ if err := dropTableColumns(sess, "drop_test", dropcols...); err != nil {
+ sess.Close()
+ t.Errorf("Unable to drop columns: %s from drop_test: %v", dropcols, err)
+ return
+ }
+ if err := sess.Commit(); err != nil {
+ sess.Close()
+ t.Errorf("unable to commit transaction: %v", err)
+ return
+ }
+ sess.Close()
+ if err := x.DropTables(new(DropTest)); err != nil {
+ t.Errorf("unable to drop table: %v", err)
+ return
+ }
+ }
+ }
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "runtime"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "code.gitea.io/gitea/modules/log"
+ "code.gitea.io/gitea/modules/queue"
+ jsoniter "github.com/json-iterator/go"
+)
+
+var (
+ prefix string
+ slowTest = 10 * time.Second
+ slowFlush = 5 * time.Second
+)
+
+// TestLogger is a logger which will write to the testing log
+type TestLogger struct {
+ log.WriterLogger
+}
+
+var writerCloser = &testLoggerWriterCloser{}
+
+type testLoggerWriterCloser struct {
+ sync.RWMutex
+ t []*testing.TB
+}
+
+func (w *testLoggerWriterCloser) setT(t *testing.TB) {
+ w.Lock()
+ w.t = append(w.t, t)
+ w.Unlock()
+}
+
+func (w *testLoggerWriterCloser) Write(p []byte) (int, error) {
+ w.RLock()
+ var t *testing.TB
+ if len(w.t) > 0 {
+ t = w.t[len(w.t)-1]
+ }
+ w.RUnlock()
+ if t != nil && *t != nil {
+ if len(p) > 0 && p[len(p)-1] == '\n' {
+ p = p[:len(p)-1]
+ }
+
+ defer func() {
+ err := recover()
+ if err == nil {
+ return
+ }
+ var errString string
+ errErr, ok := err.(error)
+ if ok {
+ errString = errErr.Error()
+ } else {
+ errString, ok = err.(string)
+ }
+ if !ok {
+ panic(err)
+ }
+ if !strings.HasPrefix(errString, "Log in goroutine after ") {
+ panic(err)
+ }
+ }()
+
+ (*t).Log(string(p))
+ return len(p), nil
+ }
+ return len(p), nil
+}
+
+func (w *testLoggerWriterCloser) Close() error {
+ w.Lock()
+ if len(w.t) > 0 {
+ w.t = w.t[:len(w.t)-1]
+ }
+ w.Unlock()
+ return nil
+}
+
+// PrintCurrentTest prints the current test to os.Stdout
+func PrintCurrentTest(t testing.TB, skip ...int) func() {
+ start := time.Now()
+ actualSkip := 1
+ if len(skip) > 0 {
+ actualSkip = skip[0]
+ }
+ _, filename, line, _ := runtime.Caller(actualSkip)
+
+ if log.CanColorStdout {
+ fmt.Fprintf(os.Stdout, "=== %s (%s:%d)\n", fmt.Formatter(log.NewColoredValue(t.Name())), strings.TrimPrefix(filename, prefix), line)
+ } else {
+ fmt.Fprintf(os.Stdout, "=== %s (%s:%d)\n", t.Name(), strings.TrimPrefix(filename, prefix), line)
+ }
+ writerCloser.setT(&t)
+ return func() {
+ took := time.Since(start)
+ if took > slowTest {
+ if log.CanColorStdout {
+ fmt.Fprintf(os.Stdout, "+++ %s is a slow test (took %v)\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgYellow)), fmt.Formatter(log.NewColoredValue(took, log.Bold, log.FgYellow)))
+ } else {
+ fmt.Fprintf(os.Stdout, "+++ %s is a slow tets (took %v)\n", t.Name(), took)
+ }
+ }
+ timer := time.AfterFunc(slowFlush, func() {
+ if log.CanColorStdout {
+ fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgRed)), slowFlush)
+ } else {
+ fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", t.Name(), slowFlush)
+ }
+ })
+ if err := queue.GetManager().FlushAll(context.Background(), -1); err != nil {
+ t.Errorf("Flushing queues failed with error %v", err)
+ }
+ timer.Stop()
+ flushTook := time.Since(start) - took
+ if flushTook > slowFlush {
+ if log.CanColorStdout {
+ fmt.Fprintf(os.Stdout, "+++ %s had a slow clean-up flush (took %v)\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgRed)), fmt.Formatter(log.NewColoredValue(flushTook, log.Bold, log.FgRed)))
+ } else {
+ fmt.Fprintf(os.Stdout, "+++ %s had a slow clean-up flush (took %v)\n", t.Name(), flushTook)
+ }
+ }
+ _ = writerCloser.Close()
+ }
+}
+
+// Printf takes a format and args and prints the string to os.Stdout
+func Printf(format string, args ...interface{}) {
+ if log.CanColorStdout {
+ for i := 0; i < len(args); i++ {
+ args[i] = log.NewColoredValue(args[i])
+ }
+ }
+ fmt.Fprintf(os.Stdout, "\t"+format, args...)
+}
+
+// NewTestLogger creates a TestLogger as a log.LoggerProvider
+func NewTestLogger() log.LoggerProvider {
+ logger := &TestLogger{}
+ logger.Colorize = log.CanColorStdout
+ logger.Level = log.TRACE
+ return logger
+}
+
+// Init inits connection writer with json config.
+// json config only need key "level".
+func (log *TestLogger) Init(config string) error {
+ json := jsoniter.ConfigCompatibleWithStandardLibrary
+ err := json.Unmarshal([]byte(config), log)
+ if err != nil {
+ return err
+ }
+ log.NewWriterLogger(writerCloser)
+ return nil
+}
+
+// Flush when log should be flushed
+func (log *TestLogger) Flush() {
+}
+
+//ReleaseReopen does nothing
+func (log *TestLogger) ReleaseReopen() error {
+ return nil
+}
+
+// GetName returns the default name for this implementation
+func (log *TestLogger) GetName() string {
+ return "test"
+}
+
+func init() {
+ log.Register("test", NewTestLogger)
+ _, filename, _, _ := runtime.Caller(0)
+ prefix = strings.TrimSuffix(filename, "integrations/testlogger.go")
+}
"xorm.io/xorm"
)
+// removeInvalidLabels looks through the database to look for comments and issue_labels
+// that refer to labels do not belong to the repository or organization that repository
+// that the issue is in
func removeInvalidLabels(x *xorm.Engine) error {
type Comment struct {
ID int64 `xorm:"pk autoincr"`
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func Test_removeInvalidLabels(t *testing.T) {
+ // Models used by the migration
+ type Comment struct {
+ ID int64 `xorm:"pk autoincr"`
+ Type int `xorm:"INDEX"`
+ IssueID int64 `xorm:"INDEX"`
+ LabelID int64
+ ShouldRemain bool // <- Flag for testing the migration
+ }
+
+ type Issue struct {
+ ID int64 `xorm:"pk autoincr"`
+ RepoID int64 `xorm:"INDEX UNIQUE(repo_index)"`
+ Index int64 `xorm:"UNIQUE(repo_index)"` // Index in one repository.
+ }
+
+ type Repository struct {
+ ID int64 `xorm:"pk autoincr"`
+ OwnerID int64 `xorm:"UNIQUE(s) index"`
+ LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"`
+ }
+
+ type Label struct {
+ ID int64 `xorm:"pk autoincr"`
+ RepoID int64 `xorm:"INDEX"`
+ OrgID int64 `xorm:"INDEX"`
+ }
+
+ type IssueLabel struct {
+ ID int64 `xorm:"pk autoincr"`
+ IssueID int64 `xorm:"UNIQUE(s)"`
+ LabelID int64 `xorm:"UNIQUE(s)"`
+ ShouldRemain bool // <- Flag for testing the migration
+ }
+
+ // load and prepare the test database
+ x, deferable := prepareTestEnv(t, 0, new(Comment), new(Issue), new(Repository), new(IssueLabel), new(Label))
+ if x == nil || t.Failed() {
+ defer deferable()
+ return
+ }
+ defer deferable()
+
+ var issueLabels []*IssueLabel
+ ilPreMigration := map[int64]*IssueLabel{}
+ ilPostMigration := map[int64]*IssueLabel{}
+
+ var comments []*Comment
+ comPreMigration := map[int64]*Comment{}
+ comPostMigration := map[int64]*Comment{}
+
+ // Get pre migration values
+ if err := x.Find(&issueLabels); err != nil {
+ t.Errorf("Unable to find issueLabels: %v", err)
+ return
+ }
+ for _, issueLabel := range issueLabels {
+ ilPreMigration[issueLabel.ID] = issueLabel
+ }
+ if err := x.Find(&comments); err != nil {
+ t.Errorf("Unable to find comments: %v", err)
+ return
+ }
+ for _, comment := range comments {
+ comPreMigration[comment.ID] = comment
+ }
+
+ // Run the migration
+ if err := removeInvalidLabels(x); err != nil {
+ t.Errorf("unable to RemoveInvalidLabels: %v", err)
+ }
+
+ // Get the post migration values
+ issueLabels = issueLabels[:0]
+ if err := x.Find(&issueLabels); err != nil {
+ t.Errorf("Unable to find issueLabels: %v", err)
+ return
+ }
+ for _, issueLabel := range issueLabels {
+ ilPostMigration[issueLabel.ID] = issueLabel
+ }
+ comments = comments[:0]
+ if err := x.Find(&comments); err != nil {
+ t.Errorf("Unable to find comments: %v", err)
+ return
+ }
+ for _, comment := range comments {
+ comPostMigration[comment.ID] = comment
+ }
+
+ // Finally test results of the migration
+ for id, comment := range comPreMigration {
+ post, ok := comPostMigration[id]
+ if ok {
+ if !comment.ShouldRemain {
+ t.Errorf("Comment[%d] remained but should have been deleted", id)
+ }
+ assert.Equal(t, comment, post)
+ } else if comment.ShouldRemain {
+ t.Errorf("Comment[%d] was deleted but should have remained", id)
+ }
+ }
+
+ for id, il := range ilPreMigration {
+ post, ok := ilPostMigration[id]
+ if ok {
+ if !il.ShouldRemain {
+ t.Errorf("IssueLabel[%d] remained but should have been deleted", id)
+ }
+ assert.Equal(t, il, post)
+ } else if il.ShouldRemain {
+ t.Errorf("IssueLabel[%d] was deleted but should have remained", id)
+ }
+ }
+
+}
"xorm.io/xorm"
)
+// deleteOrphanedIssueLabels looks through the database for issue_labels where the label no longer exists and deletes them.
func deleteOrphanedIssueLabels(x *xorm.Engine) error {
type IssueLabel struct {
ID int64 `xorm:"pk autoincr"`
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "testing"
+
+ "code.gitea.io/gitea/modules/timeutil"
+ "github.com/stretchr/testify/assert"
+)
+
+func Test_deleteOrphanedIssueLabels(t *testing.T) {
+ // Create the models used in the migration
+ type IssueLabel struct {
+ ID int64 `xorm:"pk autoincr"`
+ IssueID int64 `xorm:"UNIQUE(s)"`
+ LabelID int64 `xorm:"UNIQUE(s)"`
+ }
+
+ type Label struct {
+ ID int64 `xorm:"pk autoincr"`
+ RepoID int64 `xorm:"INDEX"`
+ OrgID int64 `xorm:"INDEX"`
+ Name string
+ Description string
+ Color string `xorm:"VARCHAR(7)"`
+ NumIssues int
+ NumClosedIssues int
+ CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
+ UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
+ }
+
+ // Prepare and load the testing database
+ x, deferable := prepareTestEnv(t, 0, new(IssueLabel), new(Label))
+ if x == nil || t.Failed() {
+ defer deferable()
+ return
+ }
+ defer deferable()
+
+ var issueLabels []*IssueLabel
+ preMigration := map[int64]*IssueLabel{}
+ postMigration := map[int64]*IssueLabel{}
+
+ // Load issue labels that exist in the database pre-migration
+ if err := x.Find(&issueLabels); err != nil {
+ assert.NoError(t, err)
+ return
+ }
+ for _, issueLabel := range issueLabels {
+ preMigration[issueLabel.ID] = issueLabel
+ }
+
+ // Run the migration
+ if err := deleteOrphanedIssueLabels(x); err != nil {
+ assert.NoError(t, err)
+ return
+ }
+
+ // Load the remaining issue-labels
+ issueLabels = issueLabels[:0]
+ if err := x.Find(&issueLabels); err != nil {
+ assert.NoError(t, err)
+ return
+ }
+ for _, issueLabel := range issueLabels {
+ postMigration[issueLabel.ID] = issueLabel
+ }
+
+ // Now test what is left
+ if _, ok := postMigration[2]; ok {
+ t.Errorf("Orphaned Label[2] survived the migration")
+ return
+ }
+
+ if _, ok := postMigration[5]; ok {
+ t.Errorf("Orphaned Label[5] survived the migration")
+ return
+ }
+
+ for id, post := range postMigration {
+ pre := preMigration[id]
+ assert.Equal(t, pre, post, "migration changed issueLabel %d", id)
+ }
+
+}
}
}
-func getEngine() (*xorm.Engine, error) {
+// GetNewEngine returns a new xorm engine from the configuration
+func GetNewEngine() (*xorm.Engine, error) {
connStr, err := setting.DBConnStr()
if err != nil {
return nil, err
// NewTestEngine sets a new test xorm.Engine
func NewTestEngine() (err error) {
- x, err = getEngine()
+ x, err = GetNewEngine()
if err != nil {
return fmt.Errorf("Connect to database: %v", err)
}
// SetEngine sets the xorm.Engine
func SetEngine() (err error) {
- x, err = getEngine()
+ x, err = GetNewEngine()
if err != nil {
return fmt.Errorf("Failed to connect to database: %v", err)
}
"time"
"github.com/go-testfixtures/testfixtures/v3"
+ "xorm.io/xorm"
"xorm.io/xorm/schemas"
)
var fixtures *testfixtures.Loader
// InitFixtures initialize test fixtures for a test database
-func InitFixtures(dir string) (err error) {
+func InitFixtures(dir string, engine ...*xorm.Engine) (err error) {
+ e := x
+ if len(engine) == 1 {
+ e = engine[0]
+ }
+
testfiles := testfixtures.Directory(dir)
dialect := "unknown"
- switch x.Dialect().URI().DBType {
+ switch e.Dialect().URI().DBType {
case schemas.POSTGRES:
dialect = "postgres"
case schemas.MYSQL:
os.Exit(1)
}
loaderOptions := []func(loader *testfixtures.Loader) error{
- testfixtures.Database(x.DB().DB),
+ testfixtures.Database(e.DB().DB),
testfixtures.Dialect(dialect),
testfixtures.DangerousSkipTestDatabaseCheck(),
testfiles,
}
- if x.Dialect().URI().DBType == schemas.POSTGRES {
+ if e.Dialect().URI().DBType == schemas.POSTGRES {
loaderOptions = append(loaderOptions, testfixtures.SkipResetSequences())
}
}
// LoadFixtures load fixtures for a test database
-func LoadFixtures() error {
+func LoadFixtures(engine ...*xorm.Engine) error {
+ e := x
+ if len(engine) == 1 {
+ e = engine[0]
+ }
var err error
// Database transaction conflicts could occur and result in ROLLBACK
// As a simple workaround, we just retry 20 times.
fmt.Printf("LoadFixtures failed after retries: %v\n", err)
}
// Now if we're running postgres we need to tell it to update the sequences
- if x.Dialect().URI().DBType == schemas.POSTGRES {
- results, err := x.QueryString(`SELECT 'SELECT SETVAL(' ||
+ if e.Dialect().URI().DBType == schemas.POSTGRES {
+ results, err := e.QueryString(`SELECT 'SELECT SETVAL(' ||
quote_literal(quote_ident(PGT.schemaname) || '.' || quote_ident(S.relname)) ||
', COALESCE(MAX(' ||quote_ident(C.attname)|| '), 1) ) FROM ' ||
quote_ident(PGT.schemaname)|| '.'||quote_ident(T.relname)|| ';'
}
for _, r := range results {
for _, value := range r {
- _, err = x.Exec(value)
+ _, err = e.Exec(value)
if err != nil {
fmt.Printf("Failed to update sequence: %s Error: %v\n", value, err)
return err
projects / gitea.git / commitdiff