diff options
Diffstat (limited to 'models/db')
| -rw-r--r-- | models/db/collation.go | 2 | ||||
| -rw-r--r-- | models/db/context.go | 8 | ||||
| -rw-r--r-- | models/db/convert.go | 16 | ||||
| -rwxr-xr-x | models/db/engine.go | 220 | ||||
| -rw-r--r-- | models/db/engine_dump.go | 33 | ||||
| -rw-r--r-- | models/db/engine_hook.go | 34 | ||||
| -rw-r--r-- | models/db/engine_init.go | 140 | ||||
| -rw-r--r-- | models/db/name.go | 2 | ||||
| -rw-r--r-- | models/db/sequence.go | 6 |
9 files changed, 242 insertions, 219 deletions
diff --git a/models/db/collation.go b/models/db/collation.go index a7db9f5442..79ade87380 100644 --- a/models/db/collation.go +++ b/models/db/collation.go @@ -140,7 +140,7 @@ func CheckCollations(x *xorm.Engine) (*CheckCollationsResult, error) { } func CheckCollationsDefaultEngine() (*CheckCollationsResult, error) { - return CheckCollations(x) + return CheckCollations(xormEngine) } func alterDatabaseCollation(x *xorm.Engine, collation string) error { diff --git a/models/db/context.go b/models/db/context.go index 171e26b933..51627712b1 100644 --- a/models/db/context.go +++ b/models/db/context.go @@ -94,7 +94,7 @@ func GetEngine(ctx context.Context) Engine { if e := getExistingEngine(ctx); e != nil { return e } - return x.Context(ctx) + return xormEngine.Context(ctx) } // getExistingEngine gets an existing db Engine/Statement from this context or returns nil @@ -155,7 +155,7 @@ func TxContext(parentCtx context.Context) (*Context, Committer, error) { return newContext(parentCtx, sess), &halfCommitter{committer: sess}, nil } - sess := x.NewSession() + sess := xormEngine.NewSession() if err := sess.Begin(); err != nil { _ = sess.Close() return nil, nil, err @@ -179,7 +179,7 @@ func WithTx(parentCtx context.Context, f func(ctx context.Context) error) error } func txWithNoCheck(parentCtx context.Context, f func(ctx context.Context) error) error { - sess := x.NewSession() + sess := xormEngine.NewSession() defer sess.Close() if err := sess.Begin(); err != nil { return err @@ -322,7 +322,7 @@ func CountByBean(ctx context.Context, bean any) (int64, error) { // TableName returns the table name according a bean object func TableName(bean any) string { - return x.TableName(bean) + return xormEngine.TableName(bean) } // InTransaction returns true if the engine is in a transaction otherwise return false diff --git a/models/db/convert.go b/models/db/convert.go index 8c124471ab..80b0f7b04b 100644 --- a/models/db/convert.go +++ b/models/db/convert.go @@ -16,30 +16,30 @@ import ( // ConvertDatabaseTable converts database and tables from utf8 to utf8mb4 if it's mysql and set ROW_FORMAT=dynamic func ConvertDatabaseTable() error { - if x.Dialect().URI().DBType != schemas.MYSQL { + if xormEngine.Dialect().URI().DBType != schemas.MYSQL { return nil } - r, err := CheckCollations(x) + r, err := CheckCollations(xormEngine) if err != nil { return err } - _, err = x.Exec(fmt.Sprintf("ALTER DATABASE `%s` CHARACTER SET utf8mb4 COLLATE %s", setting.Database.Name, r.ExpectedCollation)) + _, err = xormEngine.Exec(fmt.Sprintf("ALTER DATABASE `%s` CHARACTER SET utf8mb4 COLLATE %s", setting.Database.Name, r.ExpectedCollation)) if err != nil { return err } - tables, err := x.DBMetas() + tables, err := xormEngine.DBMetas() if err != nil { return err } for _, table := range tables { - if _, err := x.Exec(fmt.Sprintf("ALTER TABLE `%s` ROW_FORMAT=dynamic", table.Name)); err != nil { + if _, err := xormEngine.Exec(fmt.Sprintf("ALTER TABLE `%s` ROW_FORMAT=dynamic", table.Name)); err != nil { return err } - if _, err := x.Exec(fmt.Sprintf("ALTER TABLE `%s` CONVERT TO CHARACTER SET utf8mb4 COLLATE %s", table.Name, r.ExpectedCollation)); err != nil { + if _, err := xormEngine.Exec(fmt.Sprintf("ALTER TABLE `%s` CONVERT TO CHARACTER SET utf8mb4 COLLATE %s", table.Name, r.ExpectedCollation)); err != nil { return err } } @@ -49,11 +49,11 @@ func ConvertDatabaseTable() error { // ConvertVarcharToNVarchar converts database and tables from varchar to nvarchar if it's mssql func ConvertVarcharToNVarchar() error { - if x.Dialect().URI().DBType != schemas.MSSQL { + if xormEngine.Dialect().URI().DBType != schemas.MSSQL { return nil } - sess := x.NewSession() + sess := xormEngine.NewSession() defer sess.Close() res, err := sess.QuerySliceString(`SELECT 'ALTER TABLE ' + OBJECT_NAME(SC.object_id) + ' MODIFY SC.name NVARCHAR(' + CONVERT(VARCHAR(5),SC.max_length) + ')' FROM SYS.columns SC diff --git a/models/db/engine.go b/models/db/engine.go index b17188945a..91015f7038 100755 --- a/models/db/engine.go +++ b/models/db/engine.go @@ -8,17 +8,10 @@ import ( "context" "database/sql" "fmt" - "io" "reflect" "strings" - "time" - - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" "xorm.io/xorm" - "xorm.io/xorm/contexts" - "xorm.io/xorm/names" "xorm.io/xorm/schemas" _ "github.com/go-sql-driver/mysql" // Needed for the MySQL driver @@ -27,9 +20,9 @@ import ( ) var ( - x *xorm.Engine - tables []any - initFuncs []func() error + xormEngine *xorm.Engine + registeredModels []any + registeredInitFuncs []func() error ) // Engine represents a xorm engine or session. @@ -70,167 +63,38 @@ type Engine interface { // TableInfo returns table's information via an object func TableInfo(v any) (*schemas.Table, error) { - return x.TableInfo(v) -} - -// DumpTables dump tables information -func DumpTables(tables []*schemas.Table, w io.Writer, tp ...schemas.DBType) error { - return x.DumpTables(tables, w, tp...) + return xormEngine.TableInfo(v) } -// RegisterModel registers model, if initfunc provided, it will be invoked after data model sync +// RegisterModel registers model, if initFuncs provided, it will be invoked after data model sync func RegisterModel(bean any, initFunc ...func() error) { - tables = append(tables, bean) - if len(initFuncs) > 0 && initFunc[0] != nil { - initFuncs = append(initFuncs, initFunc[0]) - } -} - -func init() { - gonicNames := []string{"SSL", "UID"} - for _, name := range gonicNames { - names.LintGonicMapper[name] = true - } -} - -// newXORMEngine returns a new XORM engine from the configuration -func newXORMEngine() (*xorm.Engine, error) { - connStr, err := setting.DBConnStr() - if err != nil { - return nil, err - } - - var engine *xorm.Engine - - if setting.Database.Type.IsPostgreSQL() && len(setting.Database.Schema) > 0 { - // OK whilst we sort out our schema issues - create a schema aware postgres - registerPostgresSchemaDriver() - engine, err = xorm.NewEngine("postgresschema", connStr) - } else { - engine, err = xorm.NewEngine(setting.Database.Type.String(), connStr) - } - - if err != nil { - return nil, err + registeredModels = append(registeredModels, bean) + if len(registeredInitFuncs) > 0 && initFunc[0] != nil { + registeredInitFuncs = append(registeredInitFuncs, initFunc[0]) } - if setting.Database.Type == "mysql" { - engine.Dialect().SetParams(map[string]string{"rowFormat": "DYNAMIC"}) - } else if setting.Database.Type == "mssql" { - engine.Dialect().SetParams(map[string]string{"DEFAULT_VARCHAR": "nvarchar"}) - } - engine.SetSchema(setting.Database.Schema) - return engine, nil } // SyncAllTables sync the schemas of all tables, is required by unit test code func SyncAllTables() error { - _, err := x.StoreEngine("InnoDB").SyncWithOptions(xorm.SyncOptions{ + _, err := xormEngine.StoreEngine("InnoDB").SyncWithOptions(xorm.SyncOptions{ WarnIfDatabaseColumnMissed: true, - }, tables...) + }, registeredModels...) return err } -// InitEngine initializes the xorm.Engine and sets it as db.DefaultContext -func InitEngine(ctx context.Context) error { - xormEngine, err := newXORMEngine() - if err != nil { - if strings.Contains(err.Error(), "SQLite3 support") { - return fmt.Errorf(`sqlite3 requires: -tags sqlite,sqlite_unlock_notify%s%w`, "\n", err) - } - return fmt.Errorf("failed to connect to database: %w", err) - } - - xormEngine.SetMapper(names.GonicMapper{}) - // WARNING: for serv command, MUST remove the output to os.stdout, - // so use log file to instead print to stdout. - xormEngine.SetLogger(NewXORMLogger(setting.Database.LogSQL)) - xormEngine.ShowSQL(setting.Database.LogSQL) - xormEngine.SetMaxOpenConns(setting.Database.MaxOpenConns) - xormEngine.SetMaxIdleConns(setting.Database.MaxIdleConns) - xormEngine.SetConnMaxLifetime(setting.Database.ConnMaxLifetime) - xormEngine.SetDefaultContext(ctx) - - if setting.Database.SlowQueryThreshold > 0 { - xormEngine.AddHook(&SlowQueryHook{ - Threshold: setting.Database.SlowQueryThreshold, - Logger: log.GetLogger("xorm"), - }) - } - - SetDefaultEngine(ctx, xormEngine) - return nil -} - -// SetDefaultEngine sets the default engine for db -func SetDefaultEngine(ctx context.Context, eng *xorm.Engine) { - x = eng - DefaultContext = &Context{Context: ctx, engine: x} -} - -// UnsetDefaultEngine closes and unsets the default engine -// We hope the SetDefaultEngine and UnsetDefaultEngine can be paired, but it's impossible now, -// there are many calls to InitEngine -> SetDefaultEngine directly to overwrite the `x` and DefaultContext without close -// Global database engine related functions are all racy and there is no graceful close right now. -func UnsetDefaultEngine() { - if x != nil { - _ = x.Close() - x = nil - } - DefaultContext = nil -} - -// InitEngineWithMigration initializes a new xorm.Engine and sets it as the db.DefaultContext -// This function must never call .Sync() if the provided migration function fails. -// When called from the "doctor" command, the migration function is a version check -// that prevents the doctor from fixing anything in the database if the migration level -// is different from the expected value. -func InitEngineWithMigration(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err error) { - if err = InitEngine(ctx); err != nil { - return err - } - - if err = x.Ping(); err != nil { - return err - } - - preprocessDatabaseCollation(x) - - // We have to run migrateFunc here in case the user is re-running installation on a previously created DB. - // If we do not then table schemas will be changed and there will be conflicts when the migrations run properly. - // - // Installation should only be being re-run if users want to recover an old database. - // However, we should think carefully about should we support re-install on an installed instance, - // as there may be other problems due to secret reinitialization. - if err = migrateFunc(x); err != nil { - return fmt.Errorf("migrate: %w", err) - } - - if err = SyncAllTables(); err != nil { - return fmt.Errorf("sync database struct error: %w", err) - } - - for _, initFunc := range initFuncs { - if err := initFunc(); err != nil { - return fmt.Errorf("initFunc failed: %w", err) - } - } - - return nil -} - // NamesToBean return a list of beans or an error func NamesToBean(names ...string) ([]any, error) { beans := []any{} if len(names) == 0 { - beans = append(beans, tables...) + beans = append(beans, registeredModels...) return beans, nil } // Need to map provided names to beans... beanMap := make(map[string]any) - for _, bean := range tables { + for _, bean := range registeredModels { beanMap[strings.ToLower(reflect.Indirect(reflect.ValueOf(bean)).Type().Name())] = bean - beanMap[strings.ToLower(x.TableName(bean))] = bean - beanMap[strings.ToLower(x.TableName(bean, true))] = bean + beanMap[strings.ToLower(xormEngine.TableName(bean))] = bean + beanMap[strings.ToLower(xormEngine.TableName(bean, true))] = bean } gotBean := make(map[any]bool) @@ -247,36 +111,9 @@ func NamesToBean(names ...string) ([]any, error) { return beans, nil } -// DumpDatabase dumps all data from database according the special database SQL syntax to file system. -func DumpDatabase(filePath, dbType string) error { - var tbs []*schemas.Table - for _, t := range tables { - t, err := x.TableInfo(t) - if err != nil { - return err - } - tbs = append(tbs, t) - } - - type Version struct { - ID int64 `xorm:"pk autoincr"` - Version int64 - } - t, err := x.TableInfo(&Version{}) - if err != nil { - return err - } - tbs = append(tbs, t) - - if len(dbType) > 0 { - return x.DumpTablesToFile(tbs, filePath, schemas.DBType(dbType)) - } - return x.DumpTablesToFile(tbs, filePath) -} - // MaxBatchInsertSize returns the table's max batch insert size func MaxBatchInsertSize(bean any) int { - t, err := x.TableInfo(bean) + t, err := xormEngine.TableInfo(bean) if err != nil { return 50 } @@ -285,18 +122,18 @@ func MaxBatchInsertSize(bean any) int { // IsTableNotEmpty returns true if table has at least one record func IsTableNotEmpty(beanOrTableName any) (bool, error) { - return x.Table(beanOrTableName).Exist() + return xormEngine.Table(beanOrTableName).Exist() } // DeleteAllRecords will delete all the records of this table func DeleteAllRecords(tableName string) error { - _, err := x.Exec(fmt.Sprintf("DELETE FROM %s", tableName)) + _, err := xormEngine.Exec(fmt.Sprintf("DELETE FROM %s", tableName)) return err } // GetMaxID will return max id of the table func GetMaxID(beanOrTableName any) (maxID int64, err error) { - _, err = x.Select("MAX(id)").Table(beanOrTableName).Get(&maxID) + _, err = xormEngine.Select("MAX(id)").Table(beanOrTableName).Get(&maxID) return maxID, err } @@ -308,24 +145,3 @@ func SetLogSQL(ctx context.Context, on bool) { sess.Engine().ShowSQL(on) } } - -type SlowQueryHook struct { - Threshold time.Duration - Logger log.Logger -} - -var _ contexts.Hook = &SlowQueryHook{} - -func (SlowQueryHook) BeforeProcess(c *contexts.ContextHook) (context.Context, error) { - return c.Ctx, nil -} - -func (h *SlowQueryHook) AfterProcess(c *contexts.ContextHook) error { - if c.ExecuteTime >= h.Threshold { - // 8 is the amount of skips passed to runtime.Caller, so that in the log the correct function - // is being displayed (the function that ultimately wants to execute the query in the code) - // instead of the function of the slow query hook being called. - h.Logger.Log(8, log.WARN, "[Slow SQL Query] %s %v - %v", c.SQL, c.Args, c.ExecuteTime) - } - return nil -} diff --git a/models/db/engine_dump.go b/models/db/engine_dump.go new file mode 100644 index 0000000000..63f2d4e093 --- /dev/null +++ b/models/db/engine_dump.go @@ -0,0 +1,33 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package db + +import "xorm.io/xorm/schemas" + +// DumpDatabase dumps all data from database according the special database SQL syntax to file system. +func DumpDatabase(filePath, dbType string) error { + var tbs []*schemas.Table + for _, t := range registeredModels { + t, err := xormEngine.TableInfo(t) + if err != nil { + return err + } + tbs = append(tbs, t) + } + + type Version struct { + ID int64 `xorm:"pk autoincr"` + Version int64 + } + t, err := xormEngine.TableInfo(&Version{}) + if err != nil { + return err + } + tbs = append(tbs, t) + + if dbType != "" { + return xormEngine.DumpTablesToFile(tbs, filePath, schemas.DBType(dbType)) + } + return xormEngine.DumpTablesToFile(tbs, filePath) +} diff --git a/models/db/engine_hook.go b/models/db/engine_hook.go new file mode 100644 index 0000000000..b4c543c3dd --- /dev/null +++ b/models/db/engine_hook.go @@ -0,0 +1,34 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package db + +import ( + "context" + "time" + + "code.gitea.io/gitea/modules/log" + + "xorm.io/xorm/contexts" +) + +type SlowQueryHook struct { + Threshold time.Duration + Logger log.Logger +} + +var _ contexts.Hook = (*SlowQueryHook)(nil) + +func (*SlowQueryHook) BeforeProcess(c *contexts.ContextHook) (context.Context, error) { + return c.Ctx, nil +} + +func (h *SlowQueryHook) AfterProcess(c *contexts.ContextHook) error { + if c.ExecuteTime >= h.Threshold { + // 8 is the amount of skips passed to runtime.Caller, so that in the log the correct function + // is being displayed (the function that ultimately wants to execute the query in the code) + // instead of the function of the slow query hook being called. + h.Logger.Log(8, log.WARN, "[Slow SQL Query] %s %v - %v", c.SQL, c.Args, c.ExecuteTime) + } + return nil +} diff --git a/models/db/engine_init.go b/models/db/engine_init.go new file mode 100644 index 0000000000..da85018957 --- /dev/null +++ b/models/db/engine_init.go @@ -0,0 +1,140 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package db + +import ( + "context" + "fmt" + "strings" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + + "xorm.io/xorm" + "xorm.io/xorm/names" +) + +func init() { + gonicNames := []string{"SSL", "UID"} + for _, name := range gonicNames { + names.LintGonicMapper[name] = true + } +} + +// newXORMEngine returns a new XORM engine from the configuration +func newXORMEngine() (*xorm.Engine, error) { + connStr, err := setting.DBConnStr() + if err != nil { + return nil, err + } + + var engine *xorm.Engine + + if setting.Database.Type.IsPostgreSQL() && len(setting.Database.Schema) > 0 { + // OK whilst we sort out our schema issues - create a schema aware postgres + registerPostgresSchemaDriver() + engine, err = xorm.NewEngine("postgresschema", connStr) + } else { + engine, err = xorm.NewEngine(setting.Database.Type.String(), connStr) + } + + if err != nil { + return nil, err + } + if setting.Database.Type == "mysql" { + engine.Dialect().SetParams(map[string]string{"rowFormat": "DYNAMIC"}) + } else if setting.Database.Type == "mssql" { + engine.Dialect().SetParams(map[string]string{"DEFAULT_VARCHAR": "nvarchar"}) + } + engine.SetSchema(setting.Database.Schema) + return engine, nil +} + +// InitEngine initializes the xorm.Engine and sets it as db.DefaultContext +func InitEngine(ctx context.Context) error { + xe, err := newXORMEngine() + if err != nil { + if strings.Contains(err.Error(), "SQLite3 support") { + return fmt.Errorf(`sqlite3 requires: -tags sqlite,sqlite_unlock_notify%s%w`, "\n", err) + } + return fmt.Errorf("failed to connect to database: %w", err) + } + + xe.SetMapper(names.GonicMapper{}) + // WARNING: for serv command, MUST remove the output to os.stdout, + // so use log file to instead print to stdout. + xe.SetLogger(NewXORMLogger(setting.Database.LogSQL)) + xe.ShowSQL(setting.Database.LogSQL) + xe.SetMaxOpenConns(setting.Database.MaxOpenConns) + xe.SetMaxIdleConns(setting.Database.MaxIdleConns) + xe.SetConnMaxLifetime(setting.Database.ConnMaxLifetime) + xe.SetDefaultContext(ctx) + + if setting.Database.SlowQueryThreshold > 0 { + xe.AddHook(&SlowQueryHook{ + Threshold: setting.Database.SlowQueryThreshold, + Logger: log.GetLogger("xorm"), + }) + } + + SetDefaultEngine(ctx, xe) + return nil +} + +// SetDefaultEngine sets the default engine for db +func SetDefaultEngine(ctx context.Context, eng *xorm.Engine) { + xormEngine = eng + DefaultContext = &Context{Context: ctx, engine: xormEngine} +} + +// UnsetDefaultEngine closes and unsets the default engine +// We hope the SetDefaultEngine and UnsetDefaultEngine can be paired, but it's impossible now, +// there are many calls to InitEngine -> SetDefaultEngine directly to overwrite the `xormEngine` and DefaultContext without close +// Global database engine related functions are all racy and there is no graceful close right now. +func UnsetDefaultEngine() { + if xormEngine != nil { + _ = xormEngine.Close() + xormEngine = nil + } + DefaultContext = nil +} + +// InitEngineWithMigration initializes a new xorm.Engine and sets it as the db.DefaultContext +// This function must never call .Sync() if the provided migration function fails. +// When called from the "doctor" command, the migration function is a version check +// that prevents the doctor from fixing anything in the database if the migration level +// is different from the expected value. +func InitEngineWithMigration(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err error) { + if err = InitEngine(ctx); err != nil { + return err + } + + if err = xormEngine.Ping(); err != nil { + return err + } + + preprocessDatabaseCollation(xormEngine) + + // We have to run migrateFunc here in case the user is re-running installation on a previously created DB. + // If we do not then table schemas will be changed and there will be conflicts when the migrations run properly. + // + // Installation should only be being re-run if users want to recover an old database. + // However, we should think carefully about should we support re-install on an installed instance, + // as there may be other problems due to secret reinitialization. + if err = migrateFunc(xormEngine); err != nil { + return fmt.Errorf("migrate: %w", err) + } + + if err = SyncAllTables(); err != nil { + return fmt.Errorf("sync database struct error: %w", err) + } + + for _, initFunc := range registeredInitFuncs { + if err := initFunc(); err != nil { + return fmt.Errorf("initFunc failed: %w", err) + } + } + + return nil +} diff --git a/models/db/name.go b/models/db/name.go index 51be33a8bc..55c9dffb6a 100644 --- a/models/db/name.go +++ b/models/db/name.go @@ -16,7 +16,7 @@ var ( // ErrNameEmpty name is empty error ErrNameEmpty = util.SilentWrap{Message: "name is empty", Err: util.ErrInvalidArgument} - // AlphaDashDotPattern characters prohibited in a user name (anything except A-Za-z0-9_.-) + // AlphaDashDotPattern characters prohibited in a username (anything except A-Za-z0-9_.-) AlphaDashDotPattern = regexp.MustCompile(`[^\w-\.]`) ) diff --git a/models/db/sequence.go b/models/db/sequence.go index f49ad935de..9adc5113ac 100644 --- a/models/db/sequence.go +++ b/models/db/sequence.go @@ -17,11 +17,11 @@ func CountBadSequences(_ context.Context) (int64, error) { return 0, nil } - sess := x.NewSession() + sess := xormEngine.NewSession() defer sess.Close() var sequences []string - schema := x.Dialect().URI().Schema + schema := xormEngine.Dialect().URI().Schema sess.Engine().SetSchema("") if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__%_id_seq%' AND sequence_catalog = ?", setting.Database.Name).Find(&sequences); err != nil { @@ -38,7 +38,7 @@ func FixBadSequences(_ context.Context) error { return nil } - sess := x.NewSession() + sess := xormEngine.NewSession() defer sess.Close() if err := sess.Begin(); err != nil { return err |