Fix postgres ID sequences broken by recreate-table (#15015) (#15029)

Backport #15015

Unfortunately there is a subtle problem with recreatetable on postgres which
leads to the sequences not being renamed and not being left at 0.

Fix #14725

Signed-off-by: Andrew Thornton <art27@cantab.net>

diff --git a/cmd/doctor.go b/cmd/doctor.go
index 2ca2bb5e70b6ae221cd5947fc7f4af512d64494c..5ba0451110996b0421b543cf4986fb42f50b22fc 100644 (file)
--- a/cmd/doctor.go
+++ b/cmd/doctor.go
@@ -670,6 +670,23 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) {
                }
        }
 
+       if setting.Database.UsePostgreSQL {
+               count, err = models.CountBadSequences()
+               if err != nil {
+                       return nil, err
+               }
+               if count > 0 {
+                       if ctx.Bool("fix") {
+                               err := models.FixBadSequences()
+                               if err != nil {
+                                       return nil, err
+                               }
+                               results = append(results, fmt.Sprintf("%d sequences updated", count))
+                       } else {
+                               results = append(results, fmt.Sprintf("%d sequences with incorrect values", count))
+                       }
+               }
+       }
        //ToDo: function to recalc all counters
 
        return results, nil

diff --git a/models/consistency.go b/models/consistency.go
index fbb99ca80c8838ed1dd09ff5b27a84f578ef3674..0c62d4dc2419952176d786f9ebc41fdc70ae3b23 100644 (file)
--- a/models/consistency.go
+++ b/models/consistency.go
@@ -5,10 +5,13 @@
 package models
 
 import (
+       "fmt"
        "reflect"
+       "regexp"
        "strings"
        "testing"
 
+       "code.gitea.io/gitea/modules/setting"
        "github.com/stretchr/testify/assert"
        "xorm.io/builder"
 )
@@ -295,3 +298,61 @@ func FixNullArchivedRepository() (int64, error) {
                IsArchived: false,
        })
 }
+
+// CountBadSequences looks for broken sequences from recreate-table mistakes
+func CountBadSequences() (int64, error) {
+       if !setting.Database.UsePostgreSQL {
+               return 0, nil
+       }
+
+       sess := x.NewSession()
+       defer sess.Close()
+
+       var sequences []string
+       schema := sess.Engine().Dialect().URI().Schema
+
+       sess.Engine().SetSchema("")
+       if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__%_id_seq%' AND sequence_catalog = ?", setting.Database.Name).Find(&sequences); err != nil {
+               return 0, err
+       }
+       sess.Engine().SetSchema(schema)
+
+       return int64(len(sequences)), nil
+}
+
+// FixBadSequences fixes for broken sequences from recreate-table mistakes
+func FixBadSequences() error {
+       if !setting.Database.UsePostgreSQL {
+               return nil
+       }
+
+       sess := x.NewSession()
+       defer sess.Close()
+       if err := sess.Begin(); err != nil {
+               return err
+       }
+
+       var sequences []string
+       schema := sess.Engine().Dialect().URI().Schema
+
+       sess.Engine().SetSchema("")
+       if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__%_id_seq%' AND sequence_catalog = ?", setting.Database.Name).Find(&sequences); err != nil {
+               return err
+       }
+       sess.Engine().SetSchema(schema)
+
+       sequenceRegexp := regexp.MustCompile(`tmp_recreate__(\w+)_id_seq.*`)
+
+       for _, sequence := range sequences {
+               tableName := sequenceRegexp.FindStringSubmatch(sequence)[1]
+               newSequenceName := tableName + "_id_seq"
+               if _, err := sess.Exec(fmt.Sprintf("ALTER SEQUENCE `%s` RENAME TO `%s`", sequence, newSequenceName)); err != nil {
+                       return err
+               }
+               if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM `%s`), 1), false)", newSequenceName, tableName)); err != nil {
+                       return err
+               }
+       }
+
+       return sess.Commit()
+}

diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go
index 5cb85cc18cf6411d2f306d88bb65ec197c663b48..d16f8c7ec986ce36dc3ffd990f731b3d9e6e28ef 100644 (file)
--- a/models/migrations/migrations.go
+++ b/models/migrations/migrations.go
@@ -516,6 +516,31 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
                        return err
                }
        case setting.Database.UsePostgreSQL:
+               var originalSequences []string
+               type sequenceData struct {
+                       LastValue int  `xorm:"'last_value'"`
+                       IsCalled  bool `xorm:"'is_called'"`
+               }
+               sequenceMap := map[string]sequenceData{}
+
+               schema := sess.Engine().Dialect().URI().Schema
+               sess.Engine().SetSchema("")
+               if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE ? || '_%' AND sequence_catalog = ?", tableName, setting.Database.Name).Find(&originalSequences); err != nil {
+                       log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
+                       return err
+               }
+               sess.Engine().SetSchema(schema)
+
+               for _, sequence := range originalSequences {
+                       sequenceData := sequenceData{}
+                       if _, err := sess.Table(sequence).Cols("last_value", "is_called").Get(&sequenceData); err != nil {
+                               log.Error("Unable to get last_value and is_called from %s. Error: %v", sequence, err)
+                               return err
+                       }
+                       sequenceMap[sequence] = sequenceData
+
+               }
+
                // CASCADE causes postgres to drop all the constraints on the old table
                if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s` CASCADE", tableName)); err != nil {
                        log.Error("Unable to drop old table %s. Error: %v", tableName, err)
@@ -529,7 +554,6 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
                }
 
                var indices []string
-               schema := sess.Engine().Dialect().URI().Schema
                sess.Engine().SetSchema("")
                if err := sess.Table("pg_indexes").Cols("indexname").Where("tablename = ? ", tableName).Find(&indices); err != nil {
                        log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
@@ -545,6 +569,43 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
                        }
                }
 
+               var sequences []string
+               sess.Engine().SetSchema("")
+               if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__' || ? || '_%' AND sequence_catalog = ?", tableName, setting.Database.Name).Find(&sequences); err != nil {
+                       log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
+                       return err
+               }
+               sess.Engine().SetSchema(schema)
+
+               for _, sequence := range sequences {
+                       newSequenceName := strings.Replace(sequence, "tmp_recreate__", "", 1)
+                       if _, err := sess.Exec(fmt.Sprintf("ALTER SEQUENCE `%s` RENAME TO `%s`", sequence, newSequenceName)); err != nil {
+                               log.Error("Unable to rename %s sequence to %s. Error: %v", sequence, newSequenceName, err)
+                               return err
+                       }
+                       val, ok := sequenceMap[newSequenceName]
+                       if newSequenceName == tableName+"_id_seq" {
+                               if ok && val.LastValue != 0 {
+                                       if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', %d, %t)", newSequenceName, val.LastValue, val.IsCalled)); err != nil {
+                                               log.Error("Unable to reset %s to %d. Error: %v", newSequenceName, val, err)
+                                               return err
+                                       }
+                               } else {
+                                       // We're going to try to guess this
+                                       if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM `%s`), 1), false)", newSequenceName, tableName)); err != nil {
+                                               log.Error("Unable to reset %s. Error: %v", newSequenceName, err)
+                                               return err
+                                       }
+                               }
+                       } else if ok {
+                               if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', %d, %t)", newSequenceName, val.LastValue, val.IsCalled)); err != nil {
+                                       log.Error("Unable to reset %s to %d. Error: %v", newSequenceName, val, err)
+                                       return err
+                               }
+                       }
+
+               }
+
        case setting.Database.UseMSSQL:
                // MSSQL will drop all the constraints on the old table
                if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil {