You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

models.go 10KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370
  1. // Copyright 2014 The Gogs Authors. All rights reserved.
  2. // Copyright 2018 The Gitea Authors. All rights reserved.
  3. // Use of this source code is governed by a MIT-style
  4. // license that can be found in the LICENSE file.
  5. package models
  6. import (
  7. "database/sql"
  8. "errors"
  9. "fmt"
  10. "net/url"
  11. "os"
  12. "path"
  13. "path/filepath"
  14. "strings"
  15. "time"
  16. "code.gitea.io/gitea/modules/setting"
  17. // Needed for the MySQL driver
  18. _ "github.com/go-sql-driver/mysql"
  19. "github.com/go-xorm/core"
  20. "github.com/go-xorm/xorm"
  21. // Needed for the Postgresql driver
  22. _ "github.com/lib/pq"
  23. // Needed for the MSSSQL driver
  24. _ "github.com/denisenkom/go-mssqldb"
  25. )
  26. // Engine represents a xorm engine or session.
  27. type Engine interface {
  28. Table(tableNameOrBean interface{}) *xorm.Session
  29. Count(...interface{}) (int64, error)
  30. Decr(column string, arg ...interface{}) *xorm.Session
  31. Delete(interface{}) (int64, error)
  32. Exec(...interface{}) (sql.Result, error)
  33. Find(interface{}, ...interface{}) error
  34. Get(interface{}) (bool, error)
  35. ID(interface{}) *xorm.Session
  36. In(string, ...interface{}) *xorm.Session
  37. Incr(column string, arg ...interface{}) *xorm.Session
  38. Insert(...interface{}) (int64, error)
  39. InsertOne(interface{}) (int64, error)
  40. Iterate(interface{}, xorm.IterFunc) error
  41. Join(joinOperator string, tablename interface{}, condition string, args ...interface{}) *xorm.Session
  42. SQL(interface{}, ...interface{}) *xorm.Session
  43. Where(interface{}, ...interface{}) *xorm.Session
  44. Asc(colNames ...string) *xorm.Session
  45. }
  46. var (
  47. x *xorm.Engine
  48. supportedDatabases = []string{"mysql", "postgres", "mssql"}
  49. tables []interface{}
  50. // HasEngine specifies if we have a xorm.Engine
  51. HasEngine bool
  52. // DbCfg holds the database settings
  53. DbCfg struct {
  54. Type, Host, Name, User, Passwd, Path, SSLMode, Charset string
  55. Timeout int
  56. }
  57. // EnableSQLite3 use SQLite3
  58. EnableSQLite3 bool
  59. // EnableTiDB enable TiDB
  60. EnableTiDB bool
  61. )
  62. func init() {
  63. tables = append(tables,
  64. new(User),
  65. new(PublicKey),
  66. new(AccessToken),
  67. new(Repository),
  68. new(DeployKey),
  69. new(Collaboration),
  70. new(Access),
  71. new(Upload),
  72. new(Watch),
  73. new(Star),
  74. new(Follow),
  75. new(Action),
  76. new(Issue),
  77. new(PullRequest),
  78. new(Comment),
  79. new(Attachment),
  80. new(Label),
  81. new(IssueLabel),
  82. new(Milestone),
  83. new(Mirror),
  84. new(Release),
  85. new(LoginSource),
  86. new(Webhook),
  87. new(HookTask),
  88. new(Team),
  89. new(OrgUser),
  90. new(TeamUser),
  91. new(TeamRepo),
  92. new(Notice),
  93. new(EmailAddress),
  94. new(Notification),
  95. new(IssueUser),
  96. new(LFSMetaObject),
  97. new(TwoFactor),
  98. new(GPGKey),
  99. new(GPGKeyImport),
  100. new(RepoUnit),
  101. new(RepoRedirect),
  102. new(ExternalLoginUser),
  103. new(ProtectedBranch),
  104. new(UserOpenID),
  105. new(IssueWatch),
  106. new(CommitStatus),
  107. new(Stopwatch),
  108. new(TrackedTime),
  109. new(DeletedBranch),
  110. new(RepoIndexerStatus),
  111. new(IssueDependency),
  112. new(LFSLock),
  113. new(Reaction),
  114. new(IssueAssignees),
  115. new(U2FRegistration),
  116. new(TeamUnit),
  117. new(Review),
  118. new(OAuth2Application),
  119. new(OAuth2AuthorizationCode),
  120. new(OAuth2Grant),
  121. )
  122. gonicNames := []string{"SSL", "UID"}
  123. for _, name := range gonicNames {
  124. core.LintGonicMapper[name] = true
  125. }
  126. }
  127. // LoadConfigs loads the database settings
  128. func LoadConfigs() {
  129. sec := setting.Cfg.Section("database")
  130. DbCfg.Type = sec.Key("DB_TYPE").String()
  131. switch DbCfg.Type {
  132. case "sqlite3":
  133. setting.UseSQLite3 = true
  134. case "mysql":
  135. setting.UseMySQL = true
  136. case "postgres":
  137. setting.UsePostgreSQL = true
  138. case "tidb":
  139. setting.UseTiDB = true
  140. case "mssql":
  141. setting.UseMSSQL = true
  142. }
  143. DbCfg.Host = sec.Key("HOST").String()
  144. DbCfg.Name = sec.Key("NAME").String()
  145. DbCfg.User = sec.Key("USER").String()
  146. if len(DbCfg.Passwd) == 0 {
  147. DbCfg.Passwd = sec.Key("PASSWD").String()
  148. }
  149. DbCfg.SSLMode = sec.Key("SSL_MODE").MustString("disable")
  150. DbCfg.Charset = sec.Key("CHARSET").In("utf8", []string{"utf8", "utf8mb4"})
  151. DbCfg.Path = sec.Key("PATH").MustString(filepath.Join(setting.AppDataPath, "gitea.db"))
  152. DbCfg.Timeout = sec.Key("SQLITE_TIMEOUT").MustInt(500)
  153. }
  154. // parsePostgreSQLHostPort parses given input in various forms defined in
  155. // https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING
  156. // and returns proper host and port number.
  157. func parsePostgreSQLHostPort(info string) (string, string) {
  158. host, port := "127.0.0.1", "5432"
  159. if strings.Contains(info, ":") && !strings.HasSuffix(info, "]") {
  160. idx := strings.LastIndex(info, ":")
  161. host = info[:idx]
  162. port = info[idx+1:]
  163. } else if len(info) > 0 {
  164. host = info
  165. }
  166. return host, port
  167. }
  168. func getPostgreSQLConnectionString(dbHost, dbUser, dbPasswd, dbName, dbParam, dbsslMode string) (connStr string) {
  169. host, port := parsePostgreSQLHostPort(dbHost)
  170. if host[0] == '/' { // looks like a unix socket
  171. connStr = fmt.Sprintf("postgres://%s:%s@:%s/%s%ssslmode=%s&host=%s",
  172. url.PathEscape(dbUser), url.PathEscape(dbPasswd), port, dbName, dbParam, dbsslMode, host)
  173. } else {
  174. connStr = fmt.Sprintf("postgres://%s:%s@%s:%s/%s%ssslmode=%s",
  175. url.PathEscape(dbUser), url.PathEscape(dbPasswd), host, port, dbName, dbParam, dbsslMode)
  176. }
  177. return
  178. }
  179. // ParseMSSQLHostPort splits the host into host and port
  180. func ParseMSSQLHostPort(info string) (string, string) {
  181. host, port := "127.0.0.1", "1433"
  182. if strings.Contains(info, ":") {
  183. host = strings.Split(info, ":")[0]
  184. port = strings.Split(info, ":")[1]
  185. } else if strings.Contains(info, ",") {
  186. host = strings.Split(info, ",")[0]
  187. port = strings.TrimSpace(strings.Split(info, ",")[1])
  188. } else if len(info) > 0 {
  189. host = info
  190. }
  191. return host, port
  192. }
  193. func getEngine() (*xorm.Engine, error) {
  194. connStr := ""
  195. var Param = "?"
  196. if strings.Contains(DbCfg.Name, Param) {
  197. Param = "&"
  198. }
  199. switch DbCfg.Type {
  200. case "mysql":
  201. connType := "tcp"
  202. if DbCfg.Host[0] == '/' { // looks like a unix socket
  203. connType = "unix"
  204. }
  205. tls := DbCfg.SSLMode
  206. if tls == "disable" { // allow (Postgres-inspired) default value to work in MySQL
  207. tls = "false"
  208. }
  209. connStr = fmt.Sprintf("%s:%s@%s(%s)/%s%scharset=%s&parseTime=true&tls=%s",
  210. DbCfg.User, DbCfg.Passwd, connType, DbCfg.Host, DbCfg.Name, Param, DbCfg.Charset, tls)
  211. case "postgres":
  212. connStr = getPostgreSQLConnectionString(DbCfg.Host, DbCfg.User, DbCfg.Passwd, DbCfg.Name, Param, DbCfg.SSLMode)
  213. case "mssql":
  214. host, port := ParseMSSQLHostPort(DbCfg.Host)
  215. connStr = fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", host, port, DbCfg.Name, DbCfg.User, DbCfg.Passwd)
  216. case "sqlite3":
  217. if !EnableSQLite3 {
  218. return nil, errors.New("this binary version does not build support for SQLite3")
  219. }
  220. if err := os.MkdirAll(path.Dir(DbCfg.Path), os.ModePerm); err != nil {
  221. return nil, fmt.Errorf("Failed to create directories: %v", err)
  222. }
  223. connStr = fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d", DbCfg.Path, DbCfg.Timeout)
  224. case "tidb":
  225. if !EnableTiDB {
  226. return nil, errors.New("this binary version does not build support for TiDB")
  227. }
  228. if err := os.MkdirAll(path.Dir(DbCfg.Path), os.ModePerm); err != nil {
  229. return nil, fmt.Errorf("Failed to create directories: %v", err)
  230. }
  231. connStr = "goleveldb://" + DbCfg.Path
  232. default:
  233. return nil, fmt.Errorf("Unknown database type: %s", DbCfg.Type)
  234. }
  235. return xorm.NewEngine(DbCfg.Type, connStr)
  236. }
  237. // NewTestEngine sets a new test xorm.Engine
  238. func NewTestEngine(x *xorm.Engine) (err error) {
  239. x, err = getEngine()
  240. if err != nil {
  241. return fmt.Errorf("Connect to database: %v", err)
  242. }
  243. x.SetMapper(core.GonicMapper{})
  244. x.SetLogger(NewXORMLogger(!setting.ProdMode))
  245. x.ShowSQL(!setting.ProdMode)
  246. return x.StoreEngine("InnoDB").Sync2(tables...)
  247. }
  248. // SetEngine sets the xorm.Engine
  249. func SetEngine() (err error) {
  250. x, err = getEngine()
  251. if err != nil {
  252. return fmt.Errorf("Failed to connect to database: %v", err)
  253. }
  254. x.SetMapper(core.GonicMapper{})
  255. // WARNING: for serv command, MUST remove the output to os.stdout,
  256. // so use log file to instead print to stdout.
  257. x.SetLogger(NewXORMLogger(setting.LogSQL))
  258. x.ShowSQL(setting.LogSQL)
  259. if DbCfg.Type == "mysql" {
  260. x.SetMaxIdleConns(0)
  261. x.SetConnMaxLifetime(3 * time.Second)
  262. }
  263. return nil
  264. }
  265. // NewEngine initializes a new xorm.Engine
  266. func NewEngine(migrateFunc func(*xorm.Engine) error) (err error) {
  267. if err = SetEngine(); err != nil {
  268. return err
  269. }
  270. if err = x.Ping(); err != nil {
  271. return err
  272. }
  273. if err = migrateFunc(x); err != nil {
  274. return fmt.Errorf("migrate: %v", err)
  275. }
  276. if err = x.StoreEngine("InnoDB").Sync2(tables...); err != nil {
  277. return fmt.Errorf("sync database struct error: %v", err)
  278. }
  279. return nil
  280. }
  281. // Statistic contains the database statistics
  282. type Statistic struct {
  283. Counter struct {
  284. User, Org, PublicKey,
  285. Repo, Watch, Star, Action, Access,
  286. Issue, Comment, Oauth, Follow,
  287. Mirror, Release, LoginSource, Webhook,
  288. Milestone, Label, HookTask,
  289. Team, UpdateTask, Attachment int64
  290. }
  291. }
  292. // GetStatistic returns the database statistics
  293. func GetStatistic() (stats Statistic) {
  294. stats.Counter.User = CountUsers()
  295. stats.Counter.Org = CountOrganizations()
  296. stats.Counter.PublicKey, _ = x.Count(new(PublicKey))
  297. stats.Counter.Repo = CountRepositories(true)
  298. stats.Counter.Watch, _ = x.Count(new(Watch))
  299. stats.Counter.Star, _ = x.Count(new(Star))
  300. stats.Counter.Action, _ = x.Count(new(Action))
  301. stats.Counter.Access, _ = x.Count(new(Access))
  302. stats.Counter.Issue, _ = x.Count(new(Issue))
  303. stats.Counter.Comment, _ = x.Count(new(Comment))
  304. stats.Counter.Oauth = 0
  305. stats.Counter.Follow, _ = x.Count(new(Follow))
  306. stats.Counter.Mirror, _ = x.Count(new(Mirror))
  307. stats.Counter.Release, _ = x.Count(new(Release))
  308. stats.Counter.LoginSource = CountLoginSources()
  309. stats.Counter.Webhook, _ = x.Count(new(Webhook))
  310. stats.Counter.Milestone, _ = x.Count(new(Milestone))
  311. stats.Counter.Label, _ = x.Count(new(Label))
  312. stats.Counter.HookTask, _ = x.Count(new(HookTask))
  313. stats.Counter.Team, _ = x.Count(new(Team))
  314. stats.Counter.Attachment, _ = x.Count(new(Attachment))
  315. return
  316. }
  317. // Ping tests if database is alive
  318. func Ping() error {
  319. if x != nil {
  320. return x.Ping()
  321. }
  322. return errors.New("database not configured")
  323. }
  324. // DumpDatabase dumps all data from database according the special database SQL syntax to file system.
  325. func DumpDatabase(filePath string, dbType string) error {
  326. var tbs []*core.Table
  327. for _, t := range tables {
  328. t := x.TableInfo(t)
  329. t.Table.Name = t.Name
  330. tbs = append(tbs, t.Table)
  331. }
  332. if len(dbType) > 0 {
  333. return x.DumpTablesToFile(tbs, filePath, core.DbType(dbType))
  334. }
  335. return x.DumpTablesToFile(tbs, filePath)
  336. }