Add command to recreate tables (#12407)

Provides new command: `gitea doctor recreate-table` which will recreate
db tables and copy the old data in to the new table.

This function can be used to remove the old warning of struct defaults being
out of date.

Fix #8868
Fix #3265
Fix #8894

Signed-off-by: Andrew Thornton <art27@cantab.net>

1 files changed, 216 insertions, 0 deletions

diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go
index aca3891f3d..9b94c49cc0 100644
--- a/models/migrations/migrations.go
+++ b/models/migrations/migrations.go
@@ -7,6 +7,7 @@ package migrations
 
 import (
 	"fmt"
+	"reflect"
 	"regexp"
 	"strings"
 
@@ -327,6 +328,221 @@ Please try upgrading to a lower version first (suggested v1.6.4), then upgrade t
 	return nil
 }
 
+// RecreateTables will recreate the tables for the provided beans using the newly provided bean definition and move all data to that new table
+// WARNING: YOU MUST PROVIDE THE FULL BEAN DEFINITION
+func RecreateTables(beans ...interface{}) func(*xorm.Engine) error {
+	return func(x *xorm.Engine) error {
+		sess := x.NewSession()
+		defer sess.Close()
+		if err := sess.Begin(); err != nil {
+			return err
+		}
+		sess = sess.StoreEngine("InnoDB")
+		for _, bean := range beans {
+			log.Info("Recreating Table: %s for Bean: %s", x.TableName(bean), reflect.Indirect(reflect.ValueOf(bean)).Type().Name())
+			if err := recreateTable(sess, bean); err != nil {
+				return err
+			}
+		}
+		return sess.Commit()
+	}
+}
+
+// recreateTable will recreate the table using the newly provided bean definition and move all data to that new table
+// WARNING: YOU MUST PROVIDE THE FULL BEAN DEFINITION
+// WARNING: YOU MUST COMMIT THE SESSION AT THE END
+func recreateTable(sess *xorm.Session, bean interface{}) error {
+	// TODO: This will not work if there are foreign keys
+
+	tableName := sess.Engine().TableName(bean)
+	tempTableName := fmt.Sprintf("tmp_recreate__%s", tableName)
+
+	// We need to move the old table away and create a new one with the correct columns
+	// We will need to do this in stages to prevent data loss
+	//
+	// First create the temporary table
+	if err := sess.Table(tempTableName).CreateTable(bean); err != nil {
+		log.Error("Unable to create table %s. Error: %v", tempTableName, err)
+		return err
+	}
+
+	if err := sess.Table(tempTableName).CreateUniques(bean); err != nil {
+		log.Error("Unable to create uniques for table %s. Error: %v", tempTableName, err)
+		return err
+	}
+
+	if err := sess.Table(tempTableName).CreateIndexes(bean); err != nil {
+		log.Error("Unable to create indexes for table %s. Error: %v", tempTableName, err)
+		return err
+	}
+
+	// Work out the column names from the bean - these are the columns to select from the old table and install into the new table
+	table, err := sess.Engine().TableInfo(bean)
+	if err != nil {
+		log.Error("Unable to get table info. Error: %v", err)
+
+		return err
+	}
+	newTableColumns := table.Columns()
+	if len(newTableColumns) == 0 {
+		return fmt.Errorf("no columns in new table")
+	}
+	hasID := false
+	for _, column := range newTableColumns {
+		hasID = hasID || (column.IsPrimaryKey && column.IsAutoIncrement)
+	}
+
+	if hasID && setting.Database.UseMSSQL {
+		if _, err := sess.Exec(fmt.Sprintf("SET IDENTITY_INSERT `%s` ON", tempTableName)); err != nil {
+			log.Error("Unable to set identity insert for table %s. Error: %v", tempTableName, err)
+			return err
+		}
+	}
+
+	sqlStringBuilder := &strings.Builder{}
+	_, _ = sqlStringBuilder.WriteString("INSERT INTO `")
+	_, _ = sqlStringBuilder.WriteString(tempTableName)
+	_, _ = sqlStringBuilder.WriteString("` (`")
+	_, _ = sqlStringBuilder.WriteString(newTableColumns[0].Name)
+	_, _ = sqlStringBuilder.WriteString("`")
+	for _, column := range newTableColumns[1:] {
+		_, _ = sqlStringBuilder.WriteString(", `")
+		_, _ = sqlStringBuilder.WriteString(column.Name)
+		_, _ = sqlStringBuilder.WriteString("`")
+	}
+	_, _ = sqlStringBuilder.WriteString(")")
+	_, _ = sqlStringBuilder.WriteString(" SELECT ")
+	if newTableColumns[0].Default != "" {
+		_, _ = sqlStringBuilder.WriteString("COALESCE(`")
+		_, _ = sqlStringBuilder.WriteString(newTableColumns[0].Name)
+		_, _ = sqlStringBuilder.WriteString("`, ")
+		_, _ = sqlStringBuilder.WriteString(newTableColumns[0].Default)
+		_, _ = sqlStringBuilder.WriteString(")")
+	} else {
+		_, _ = sqlStringBuilder.WriteString("`")
+		_, _ = sqlStringBuilder.WriteString(newTableColumns[0].Name)
+		_, _ = sqlStringBuilder.WriteString("`")
+	}
+
+	for _, column := range newTableColumns[1:] {
+		if column.Default != "" {
+			_, _ = sqlStringBuilder.WriteString(", COALESCE(`")
+			_, _ = sqlStringBuilder.WriteString(column.Name)
+			_, _ = sqlStringBuilder.WriteString("`, ")
+			_, _ = sqlStringBuilder.WriteString(column.Default)
+			_, _ = sqlStringBuilder.WriteString(")")
+		} else {
+			_, _ = sqlStringBuilder.WriteString(", `")
+			_, _ = sqlStringBuilder.WriteString(column.Name)
+			_, _ = sqlStringBuilder.WriteString("`")
+		}
+	}
+	_, _ = sqlStringBuilder.WriteString(" FROM `")
+	_, _ = sqlStringBuilder.WriteString(tableName)
+	_, _ = sqlStringBuilder.WriteString("`")
+
+	if _, err := sess.Exec(sqlStringBuilder.String()); err != nil {
+		log.Error("Unable to set copy data in to temp table %s. Error: %v", tempTableName, err)
+		return err
+	}
+
+	if hasID && setting.Database.UseMSSQL {
+		if _, err := sess.Exec(fmt.Sprintf("SET IDENTITY_INSERT `%s` OFF", tempTableName)); err != nil {
+			log.Error("Unable to switch off identity insert for table %s. Error: %v", tempTableName, err)
+			return err
+		}
+	}
+
+	switch {
+	case setting.Database.UseSQLite3:
+		// SQLite will drop all the constraints on the old table
+		if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil {
+			log.Error("Unable to drop old table %s. Error: %v", tableName, err)
+			return err
+		}
+
+		if err := sess.Table(tempTableName).DropIndexes(bean); err != nil {
+			log.Error("Unable to drop indexes on temporary table %s. Error: %v", tempTableName, err)
+			return err
+		}
+
+		if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` RENAME TO `%s`", tempTableName, tableName)); err != nil {
+			log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
+			return err
+		}
+
+		if err := sess.Table(tableName).CreateIndexes(bean); err != nil {
+			log.Error("Unable to recreate indexes on table %s. Error: %v", tableName, err)
+			return err
+		}
+
+		if err := sess.Table(tableName).CreateUniques(bean); err != nil {
+			log.Error("Unable to recreate uniques on table %s. Error: %v", tableName, err)
+			return err
+		}
+
+	case setting.Database.UseMySQL:
+		// MySQL will drop all the constraints on the old table
+		if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil {
+			log.Error("Unable to drop old table %s. Error: %v", tableName, err)
+			return err
+		}
+
+		// SQLite and MySQL will move all the constraints from the temporary table to the new table
+		if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` RENAME TO `%s`", tempTableName, tableName)); err != nil {
+			log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
+			return err
+		}
+	case setting.Database.UsePostgreSQL:
+		// CASCADE causes postgres to drop all the constraints on the old table
+		if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s` CASCADE", tableName)); err != nil {
+			log.Error("Unable to drop old table %s. Error: %v", tableName, err)
+			return err
+		}
+
+		// CASCADE causes postgres to move all the constraints from the temporary table to the new table
+		if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` RENAME TO `%s`", tempTableName, tableName)); err != nil {
+			log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
+			return err
+		}
+
+		var indices []string
+		schema := sess.Engine().Dialect().URI().Schema
+		sess.Engine().SetSchema("")
+		if err := sess.Table("pg_indexes").Cols("indexname").Where("tablename = ? ", tableName).Find(&indices); err != nil {
+			log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
+			return err
+		}
+		sess.Engine().SetSchema(schema)
+
+		for _, index := range indices {
+			newIndexName := strings.Replace(index, "tmp_recreate__", "", 1)
+			if _, err := sess.Exec(fmt.Sprintf("ALTER INDEX `%s` RENAME TO `%s`", index, newIndexName)); err != nil {
+				log.Error("Unable to rename %s to %s. Error: %v", index, newIndexName, err)
+				return err
+			}
+		}
+
+	case setting.Database.UseMSSQL:
+		// MSSQL will drop all the constraints on the old table
+		if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil {
+			log.Error("Unable to drop old table %s. Error: %v", tableName, err)
+			return err
+		}
+
+		// MSSQL sp_rename will move all the constraints from the temporary table to the new table
+		if _, err := sess.Exec(fmt.Sprintf("sp_rename `%s`,`%s`", tempTableName, tableName)); err != nil {
+			log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
+			return err
+		}
+
+	default:
+		log.Fatal("Unrecognized DB")
+	}
+	return nil
+}
+
+// WARNING: YOU MUST COMMIT THE SESSION AT THE END
 func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...string) (err error) {
 	if tableName == "" || len(columnNames) == 0 {
 		return nil