You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

migrations.go 31KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032
  1. // Copyright 2015 The Gogs Authors. All rights reserved.
  2. // Copyright 2017 The Gitea Authors. All rights reserved.
  3. // Use of this source code is governed by a MIT-style
  4. // license that can be found in the LICENSE file.
  5. package migrations
  6. import (
  7. "bytes"
  8. "encoding/json"
  9. "fmt"
  10. "io/ioutil"
  11. "os"
  12. "path"
  13. "path/filepath"
  14. "regexp"
  15. "strings"
  16. "time"
  17. "code.gitea.io/gitea/modules/generate"
  18. "code.gitea.io/gitea/modules/log"
  19. "code.gitea.io/gitea/modules/setting"
  20. "github.com/go-xorm/xorm"
  21. gouuid "github.com/satori/go.uuid"
  22. "github.com/unknwon/com"
  23. ini "gopkg.in/ini.v1"
  24. )
  25. const minDBVersion = 4
  26. // Migration describes on migration from lower version to high version
  27. type Migration interface {
  28. Description() string
  29. Migrate(*xorm.Engine) error
  30. }
  31. type migration struct {
  32. description string
  33. migrate func(*xorm.Engine) error
  34. }
  35. // NewMigration creates a new migration
  36. func NewMigration(desc string, fn func(*xorm.Engine) error) Migration {
  37. return &migration{desc, fn}
  38. }
  39. // Description returns the migration's description
  40. func (m *migration) Description() string {
  41. return m.description
  42. }
  43. // Migrate executes the migration
  44. func (m *migration) Migrate(x *xorm.Engine) error {
  45. return m.migrate(x)
  46. }
  47. // Version describes the version table. Should have only one row with id==1
  48. type Version struct {
  49. ID int64 `xorm:"pk autoincr"`
  50. Version int64
  51. }
  52. func emptyMigration(x *xorm.Engine) error {
  53. return nil
  54. }
  55. // This is a sequence of migrations. Add new migrations to the bottom of the list.
  56. // If you want to "retire" a migration, remove it from the top of the list and
  57. // update minDBVersion accordingly
  58. var migrations = []Migration{
  59. // v0 -> v4: before 0.6.0 -> 0.7.33
  60. NewMigration("fix locale file load panic", fixLocaleFileLoadPanic), // V4 -> V5:v0.6.0
  61. NewMigration("trim action compare URL prefix", trimCommitActionAppURLPrefix), // V5 -> V6:v0.6.3
  62. NewMigration("generate issue-label from issue", issueToIssueLabel), // V6 -> V7:v0.6.4
  63. NewMigration("refactor attachment table", attachmentRefactor), // V7 -> V8:v0.6.4
  64. NewMigration("rename pull request fields", renamePullRequestFields), // V8 -> V9:v0.6.16
  65. NewMigration("clean up migrate repo info", cleanUpMigrateRepoInfo), // V9 -> V10:v0.6.20
  66. NewMigration("generate rands and salt for organizations", generateOrgRandsAndSalt), // V10 -> V11:v0.8.5
  67. NewMigration("convert date to unix timestamp", convertDateToUnix), // V11 -> V12:v0.9.2
  68. NewMigration("convert LDAP UseSSL option to SecurityProtocol", ldapUseSSLToSecurityProtocol), // V12 -> V13:v0.9.37
  69. // v13 -> v14:v0.9.87
  70. NewMigration("set comment updated with created", setCommentUpdatedWithCreated),
  71. // v14 -> v15
  72. NewMigration("create user column diff view style", createUserColumnDiffViewStyle),
  73. // v15 -> v16
  74. NewMigration("create user column allow create organization", createAllowCreateOrganizationColumn),
  75. // V16 -> v17
  76. NewMigration("create repo unit table and add units for all repos", addUnitsToTables),
  77. // v17 -> v18
  78. NewMigration("set protect branches updated with created", setProtectedBranchUpdatedWithCreated),
  79. // v18 -> v19
  80. NewMigration("add external login user", addExternalLoginUser),
  81. // v19 -> v20
  82. NewMigration("generate and migrate Git hooks", generateAndMigrateGitHooks),
  83. // v20 -> v21
  84. NewMigration("use new avatar path name for security reason", useNewNameAvatars),
  85. // v21 -> v22
  86. NewMigration("rewrite authorized_keys file via new format", useNewPublickeyFormat),
  87. // v22 -> v23
  88. NewMigration("generate and migrate wiki Git hooks", generateAndMigrateWikiGitHooks),
  89. // v23 -> v24
  90. NewMigration("add user openid table", addUserOpenID),
  91. // v24 -> v25
  92. NewMigration("change the key_id and primary_key_id type", changeGPGKeysColumns),
  93. // v25 -> v26
  94. NewMigration("add show field in user openid table", addUserOpenIDShow),
  95. // v26 -> v27
  96. NewMigration("generate and migrate repo and wiki Git hooks", generateAndMigrateGitHookChains),
  97. // v27 -> v28
  98. NewMigration("change mirror interval from hours to time.Duration", convertIntervalToDuration),
  99. // v28 -> v29
  100. NewMigration("add field for repo size", addRepoSize),
  101. // v29 -> v30
  102. NewMigration("add commit status table", addCommitStatus),
  103. // v30 -> 31
  104. NewMigration("add primary key to external login user", addExternalLoginUserPK),
  105. // v31 -> 32
  106. NewMigration("add field for login source synchronization", addLoginSourceSyncEnabledColumn),
  107. // v32 -> v33
  108. NewMigration("add units for team", addUnitsToRepoTeam),
  109. // v33 -> v34
  110. NewMigration("remove columns from action", removeActionColumns),
  111. // v34 -> v35
  112. NewMigration("give all units to owner teams", giveAllUnitsToOwnerTeams),
  113. // v35 -> v36
  114. NewMigration("adds comment to an action", addCommentIDToAction),
  115. // v36 -> v37
  116. NewMigration("regenerate git hooks", regenerateGitHooks36),
  117. // v37 -> v38
  118. NewMigration("unescape user full names", unescapeUserFullNames),
  119. // v38 -> v39
  120. NewMigration("remove commits and settings unit types", removeCommitsUnitType),
  121. // v39 -> v40
  122. NewMigration("add tags to releases and sync existing repositories", releaseAddColumnIsTagAndSyncTags),
  123. // v40 -> v41
  124. NewMigration("fix protected branch can push value to false", fixProtectedBranchCanPushValue),
  125. // v41 -> v42
  126. NewMigration("remove duplicate unit types", removeDuplicateUnitTypes),
  127. // v42 -> v43
  128. NewMigration("empty step", emptyMigration),
  129. // v43 -> v44
  130. NewMigration("empty step", emptyMigration),
  131. // v44 -> v45
  132. NewMigration("empty step", emptyMigration),
  133. // v45 -> v46
  134. NewMigration("remove index column from repo_unit table", removeIndexColumnFromRepoUnitTable),
  135. // v46 -> v47
  136. NewMigration("remove organization watch repositories", removeOrganizationWatchRepo),
  137. // v47 -> v48
  138. NewMigration("add deleted branches", addDeletedBranch),
  139. // v48 -> v49
  140. NewMigration("add repo indexer status", addRepoIndexerStatus),
  141. // v49 -> v50
  142. NewMigration("adds time tracking and stopwatches", addTimetracking),
  143. // v50 -> v51
  144. NewMigration("migrate protected branch struct", migrateProtectedBranchStruct),
  145. // v51 -> v52
  146. NewMigration("add default value to user prohibit_login", addDefaultValueToUserProhibitLogin),
  147. // v52 -> v53
  148. NewMigration("add lfs lock table", addLFSLock),
  149. // v53 -> v54
  150. NewMigration("add reactions", addReactions),
  151. // v54 -> v55
  152. NewMigration("add pull request options", addPullRequestOptions),
  153. // v55 -> v56
  154. NewMigration("add writable deploy keys", addModeToDeploKeys),
  155. // v56 -> v57
  156. NewMigration("remove is_owner, num_teams columns from org_user", removeIsOwnerColumnFromOrgUser),
  157. // v57 -> v58
  158. NewMigration("add closed_unix column for issues", addIssueClosedTime),
  159. // v58 -> v59
  160. NewMigration("add label descriptions", addLabelsDescriptions),
  161. // v59 -> v60
  162. NewMigration("add merge whitelist for protected branches", addProtectedBranchMergeWhitelist),
  163. // v60 -> v61
  164. NewMigration("add is_fsck_enabled column for repos", addFsckEnabledToRepo),
  165. // v61 -> v62
  166. NewMigration("add size column for attachments", addSizeToAttachment),
  167. // v62 -> v63
  168. NewMigration("add last used passcode column for TOTP", addLastUsedPasscodeTOTP),
  169. // v63 -> v64
  170. NewMigration("add language column for user setting", addLanguageSetting),
  171. // v64 -> v65
  172. NewMigration("add multiple assignees", addMultipleAssignees),
  173. // v65 -> v66
  174. NewMigration("add u2f", addU2FReg),
  175. // v66 -> v67
  176. NewMigration("add login source id column for public_key table", addLoginSourceIDToPublicKeyTable),
  177. // v67 -> v68
  178. NewMigration("remove stale watches", removeStaleWatches),
  179. // v68 -> V69
  180. NewMigration("Reformat and remove incorrect topics", reformatAndRemoveIncorrectTopics),
  181. // v69 -> v70
  182. NewMigration("move team units to team_unit table", moveTeamUnitsToTeamUnitTable),
  183. // v70 -> v71
  184. NewMigration("add issue_dependencies", addIssueDependencies),
  185. // v71 -> v72
  186. NewMigration("protect each scratch token", addScratchHash),
  187. // v72 -> v73
  188. NewMigration("add review", addReview),
  189. // v73 -> v74
  190. NewMigration("add must_change_password column for users table", addMustChangePassword),
  191. // v74 -> v75
  192. NewMigration("add approval whitelists to protected branches", addApprovalWhitelistsToProtectedBranches),
  193. // v75 -> v76
  194. NewMigration("clear nonused data which not deleted when user was deleted", clearNonusedData),
  195. // v76 -> v77
  196. NewMigration("add pull request rebase with merge commit", addPullRequestRebaseWithMerge),
  197. // v77 -> v78
  198. NewMigration("add theme to users", addUserDefaultTheme),
  199. // v78 -> v79
  200. NewMigration("rename repo is_bare to repo is_empty", renameRepoIsBareToIsEmpty),
  201. // v79 -> v80
  202. NewMigration("add can close issues via commit in any branch", addCanCloseIssuesViaCommitInAnyBranch),
  203. // v80 -> v81
  204. NewMigration("add is locked to issues", addIsLockedToIssues),
  205. // v81 -> v82
  206. NewMigration("update U2F counter type", changeU2FCounterType),
  207. // v82 -> v83
  208. NewMigration("hot fix for wrong release sha1 on release table", fixReleaseSha1OnReleaseTable),
  209. // v83 -> v84
  210. NewMigration("add uploader id for table attachment", addUploaderIDForAttachment),
  211. // v84 -> v85
  212. NewMigration("add table to store original imported gpg keys", addGPGKeyImport),
  213. // v85 -> v86
  214. NewMigration("hash application token", hashAppToken),
  215. // v86 -> v87
  216. NewMigration("add http method to webhook", addHTTPMethodToWebhook),
  217. // v87 -> v88
  218. NewMigration("add avatar field to repository", addAvatarFieldToRepository),
  219. // v88 -> v89
  220. NewMigration("add commit status context field to commit_status", addCommitStatusContext),
  221. // v89 -> v90
  222. NewMigration("add original author/url migration info to issues, comments, and repo ", addOriginalMigrationInfo),
  223. // v90 -> v91
  224. NewMigration("change length of some repository columns", changeSomeColumnsLengthOfRepo),
  225. // v91 -> v92
  226. NewMigration("add index on owner_id of repository and type, review_id of comment", addIndexOnRepositoryAndComment),
  227. // v92 -> v93
  228. NewMigration("remove orphaned repository index statuses", removeLingeringIndexStatus),
  229. }
  230. // Migrate database to current version
  231. func Migrate(x *xorm.Engine) error {
  232. if err := x.Sync(new(Version)); err != nil {
  233. return fmt.Errorf("sync: %v", err)
  234. }
  235. currentVersion := &Version{ID: 1}
  236. has, err := x.Get(currentVersion)
  237. if err != nil {
  238. return fmt.Errorf("get: %v", err)
  239. } else if !has {
  240. // If the version record does not exist we think
  241. // it is a fresh installation and we can skip all migrations.
  242. currentVersion.ID = 0
  243. currentVersion.Version = int64(minDBVersion + len(migrations))
  244. if _, err = x.InsertOne(currentVersion); err != nil {
  245. return fmt.Errorf("insert: %v", err)
  246. }
  247. }
  248. v := currentVersion.Version
  249. if minDBVersion > v {
  250. log.Fatal(`Gitea no longer supports auto-migration from your previously installed version.
  251. Please try to upgrade to a lower version (>= v0.6.0) first, then upgrade to current version.`)
  252. return nil
  253. }
  254. if int(v-minDBVersion) > len(migrations) {
  255. // User downgraded Gitea.
  256. currentVersion.Version = int64(len(migrations) + minDBVersion)
  257. _, err = x.ID(1).Update(currentVersion)
  258. return err
  259. }
  260. for i, m := range migrations[v-minDBVersion:] {
  261. log.Info("Migration[%d]: %s", v+int64(i), m.Description())
  262. if err = m.Migrate(x); err != nil {
  263. return fmt.Errorf("do migrate: %v", err)
  264. }
  265. currentVersion.Version = v + int64(i) + 1
  266. if _, err = x.ID(1).Update(currentVersion); err != nil {
  267. return err
  268. }
  269. }
  270. return nil
  271. }
  272. func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...string) (err error) {
  273. if tableName == "" || len(columnNames) == 0 {
  274. return nil
  275. }
  276. // TODO: This will not work if there are foreign keys
  277. switch {
  278. case setting.UseSQLite3:
  279. // First drop the indexes on the columns
  280. res, errIndex := sess.Query(fmt.Sprintf("PRAGMA index_list(`%s`)", tableName))
  281. if errIndex != nil {
  282. return errIndex
  283. }
  284. for _, row := range res {
  285. indexName := row["name"]
  286. indexRes, err := sess.Query(fmt.Sprintf("PRAGMA index_info(`%s`)", indexName))
  287. if err != nil {
  288. return err
  289. }
  290. if len(indexRes) != 1 {
  291. continue
  292. }
  293. indexColumn := string(indexRes[0]["name"])
  294. for _, name := range columnNames {
  295. if name == indexColumn {
  296. _, err := sess.Exec(fmt.Sprintf("DROP INDEX `%s`", indexName))
  297. if err != nil {
  298. return err
  299. }
  300. }
  301. }
  302. }
  303. // Here we need to get the columns from the original table
  304. sql := fmt.Sprintf("SELECT sql FROM sqlite_master WHERE tbl_name='%s' and type='table'", tableName)
  305. res, err := sess.Query(sql)
  306. if err != nil {
  307. return err
  308. }
  309. tableSQL := string(res[0]["sql"])
  310. // Separate out the column definitions
  311. tableSQL = tableSQL[strings.Index(tableSQL, "("):]
  312. // Remove the required columnNames
  313. for _, name := range columnNames {
  314. tableSQL = regexp.MustCompile(regexp.QuoteMeta("`"+name+"`")+"[^`,)]*?[,)]").ReplaceAllString(tableSQL, "")
  315. }
  316. // Ensure the query is ended properly
  317. tableSQL = strings.TrimSpace(tableSQL)
  318. if tableSQL[len(tableSQL)-1] != ')' {
  319. if tableSQL[len(tableSQL)-1] == ',' {
  320. tableSQL = tableSQL[:len(tableSQL)-1]
  321. }
  322. tableSQL += ")"
  323. }
  324. // Find all the columns in the table
  325. columns := regexp.MustCompile("`([^`]*)`").FindAllString(tableSQL, -1)
  326. tableSQL = fmt.Sprintf("CREATE TABLE `new_%s_new` ", tableName) + tableSQL
  327. if _, err := sess.Exec(tableSQL); err != nil {
  328. return err
  329. }
  330. // Now restore the data
  331. columnsSeparated := strings.Join(columns, ",")
  332. insertSQL := fmt.Sprintf("INSERT INTO `new_%s_new` (%s) SELECT %s FROM %s", tableName, columnsSeparated, columnsSeparated, tableName)
  333. if _, err := sess.Exec(insertSQL); err != nil {
  334. return err
  335. }
  336. // Now drop the old table
  337. if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil {
  338. return err
  339. }
  340. // Rename the table
  341. if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `new_%s_new` RENAME TO `%s`", tableName, tableName)); err != nil {
  342. return err
  343. }
  344. case setting.UsePostgreSQL:
  345. cols := ""
  346. for _, col := range columnNames {
  347. if cols != "" {
  348. cols += ", "
  349. }
  350. cols += "DROP COLUMN `" + col + "` CASCADE"
  351. }
  352. if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` %s", tableName, cols)); err != nil {
  353. return fmt.Errorf("Drop table `%s` columns %v: %v", tableName, columnNames, err)
  354. }
  355. case setting.UseMySQL, setting.UseTiDB:
  356. // Drop indexes on columns first
  357. sql := fmt.Sprintf("SHOW INDEX FROM %s WHERE column_name IN ('%s')", tableName, strings.Join(columnNames, "','"))
  358. res, err := sess.Query(sql)
  359. if err != nil {
  360. return err
  361. }
  362. for _, index := range res {
  363. indexName := index["column_name"]
  364. _, err := sess.Exec(fmt.Sprintf("DROP INDEX `%s` ON `%s`", indexName, tableName))
  365. if err != nil {
  366. return err
  367. }
  368. }
  369. // Now drop the columns
  370. cols := ""
  371. for _, col := range columnNames {
  372. if cols != "" {
  373. cols += ", "
  374. }
  375. cols += "DROP COLUMN `" + col + "`"
  376. }
  377. if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` %s", tableName, cols)); err != nil {
  378. return fmt.Errorf("Drop table `%s` columns %v: %v", tableName, columnNames, err)
  379. }
  380. case setting.UseMSSQL:
  381. cols := ""
  382. for _, col := range columnNames {
  383. if cols != "" {
  384. cols += ", "
  385. }
  386. cols += "`" + strings.ToLower(col) + "`"
  387. }
  388. sql := fmt.Sprintf("SELECT Name FROM SYS.DEFAULT_CONSTRAINTS WHERE PARENT_OBJECT_ID = OBJECT_ID('%[1]s') AND PARENT_COLUMN_ID IN (SELECT column_id FROM sys.columns WHERE lower(NAME) IN (%[2]s) AND object_id = OBJECT_ID('%[1]s'))",
  389. tableName, strings.Replace(cols, "`", "'", -1))
  390. constraints := make([]string, 0)
  391. if err := sess.SQL(sql).Find(&constraints); err != nil {
  392. sess.Rollback()
  393. return fmt.Errorf("Find constraints: %v", err)
  394. }
  395. for _, constraint := range constraints {
  396. if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` DROP CONSTRAINT `%s`", tableName, constraint)); err != nil {
  397. sess.Rollback()
  398. return fmt.Errorf("Drop table `%s` constraint `%s`: %v", tableName, constraint, err)
  399. }
  400. }
  401. if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` DROP COLUMN %s", tableName, cols)); err != nil {
  402. sess.Rollback()
  403. return fmt.Errorf("Drop table `%s` columns %v: %v", tableName, columnNames, err)
  404. }
  405. return sess.Commit()
  406. default:
  407. log.Fatal("Unrecognized DB")
  408. }
  409. return nil
  410. }
  411. func fixLocaleFileLoadPanic(_ *xorm.Engine) error {
  412. cfg, err := ini.Load(setting.CustomConf)
  413. if err != nil {
  414. return fmt.Errorf("load custom config: %v", err)
  415. }
  416. cfg.DeleteSection("i18n")
  417. if err = cfg.SaveTo(setting.CustomConf); err != nil {
  418. return fmt.Errorf("save custom config: %v", err)
  419. }
  420. setting.Langs = strings.Split(strings.Replace(strings.Join(setting.Langs, ","), "fr-CA", "fr-FR", 1), ",")
  421. return nil
  422. }
  423. func trimCommitActionAppURLPrefix(x *xorm.Engine) error {
  424. type PushCommit struct {
  425. Sha1 string
  426. Message string
  427. AuthorEmail string
  428. AuthorName string
  429. }
  430. type PushCommits struct {
  431. Len int
  432. Commits []*PushCommit
  433. CompareURL string `json:"CompareUrl"`
  434. }
  435. type Action struct {
  436. ID int64 `xorm:"pk autoincr"`
  437. Content string `xorm:"TEXT"`
  438. }
  439. results, err := x.Query("SELECT `id`,`content` FROM `action` WHERE `op_type`=?", 5)
  440. if err != nil {
  441. return fmt.Errorf("select commit actions: %v", err)
  442. }
  443. sess := x.NewSession()
  444. defer sess.Close()
  445. if err = sess.Begin(); err != nil {
  446. return err
  447. }
  448. var pushCommits *PushCommits
  449. for _, action := range results {
  450. actID := com.StrTo(string(action["id"])).MustInt64()
  451. if actID == 0 {
  452. continue
  453. }
  454. pushCommits = new(PushCommits)
  455. if err = json.Unmarshal(action["content"], pushCommits); err != nil {
  456. return fmt.Errorf("unmarshal action content[%d]: %v", actID, err)
  457. }
  458. infos := strings.Split(pushCommits.CompareURL, "/")
  459. if len(infos) <= 4 {
  460. continue
  461. }
  462. pushCommits.CompareURL = strings.Join(infos[len(infos)-4:], "/")
  463. p, err := json.Marshal(pushCommits)
  464. if err != nil {
  465. return fmt.Errorf("marshal action content[%d]: %v", actID, err)
  466. }
  467. if _, err = sess.ID(actID).Update(&Action{
  468. Content: string(p),
  469. }); err != nil {
  470. return fmt.Errorf("update action[%d]: %v", actID, err)
  471. }
  472. }
  473. return sess.Commit()
  474. }
  475. func issueToIssueLabel(x *xorm.Engine) error {
  476. type IssueLabel struct {
  477. ID int64 `xorm:"pk autoincr"`
  478. IssueID int64 `xorm:"UNIQUE(s)"`
  479. LabelID int64 `xorm:"UNIQUE(s)"`
  480. }
  481. issueLabels := make([]*IssueLabel, 0, 50)
  482. results, err := x.Query("SELECT `id`,`label_ids` FROM `issue`")
  483. if err != nil {
  484. if strings.Contains(err.Error(), "no such column") ||
  485. strings.Contains(err.Error(), "Unknown column") {
  486. return nil
  487. }
  488. return fmt.Errorf("select issues: %v", err)
  489. }
  490. for _, issue := range results {
  491. issueID := com.StrTo(issue["id"]).MustInt64()
  492. // Just in case legacy code can have duplicated IDs for same label.
  493. mark := make(map[int64]bool)
  494. for _, idStr := range strings.Split(string(issue["label_ids"]), "|") {
  495. labelID := com.StrTo(strings.TrimPrefix(idStr, "$")).MustInt64()
  496. if labelID == 0 || mark[labelID] {
  497. continue
  498. }
  499. mark[labelID] = true
  500. issueLabels = append(issueLabels, &IssueLabel{
  501. IssueID: issueID,
  502. LabelID: labelID,
  503. })
  504. }
  505. }
  506. sess := x.NewSession()
  507. defer sess.Close()
  508. if err = sess.Begin(); err != nil {
  509. return err
  510. }
  511. if err = sess.Sync2(new(IssueLabel)); err != nil {
  512. return fmt.Errorf("Sync2: %v", err)
  513. } else if _, err = sess.Insert(issueLabels); err != nil {
  514. return fmt.Errorf("insert issue-labels: %v", err)
  515. }
  516. return sess.Commit()
  517. }
  518. func attachmentRefactor(x *xorm.Engine) error {
  519. type Attachment struct {
  520. ID int64 `xorm:"pk autoincr"`
  521. UUID string `xorm:"uuid INDEX"`
  522. // For rename purpose.
  523. Path string `xorm:"-"`
  524. NewPath string `xorm:"-"`
  525. }
  526. results, err := x.Query("SELECT * FROM `attachment`")
  527. if err != nil {
  528. return fmt.Errorf("select attachments: %v", err)
  529. }
  530. attachments := make([]*Attachment, 0, len(results))
  531. for _, attach := range results {
  532. if !com.IsExist(string(attach["path"])) {
  533. // If the attachment is already missing, there is no point to update it.
  534. continue
  535. }
  536. attachments = append(attachments, &Attachment{
  537. ID: com.StrTo(attach["id"]).MustInt64(),
  538. UUID: gouuid.NewV4().String(),
  539. Path: string(attach["path"]),
  540. })
  541. }
  542. sess := x.NewSession()
  543. defer sess.Close()
  544. if err = sess.Begin(); err != nil {
  545. return err
  546. }
  547. if err = sess.Sync2(new(Attachment)); err != nil {
  548. return fmt.Errorf("Sync2: %v", err)
  549. }
  550. // Note: Roll back for rename can be a dead loop,
  551. // so produces a backup file.
  552. var buf bytes.Buffer
  553. buf.WriteString("# old path -> new path\n")
  554. // Update database first because this is where error happens the most often.
  555. for _, attach := range attachments {
  556. if _, err = sess.ID(attach.ID).Update(attach); err != nil {
  557. return err
  558. }
  559. attach.NewPath = path.Join(setting.AttachmentPath, attach.UUID[0:1], attach.UUID[1:2], attach.UUID)
  560. buf.WriteString(attach.Path)
  561. buf.WriteString("\t")
  562. buf.WriteString(attach.NewPath)
  563. buf.WriteString("\n")
  564. }
  565. // Then rename attachments.
  566. isSucceed := true
  567. defer func() {
  568. if isSucceed {
  569. return
  570. }
  571. dumpPath := path.Join(setting.LogRootPath, "attachment_path.dump")
  572. ioutil.WriteFile(dumpPath, buf.Bytes(), 0666)
  573. log.Info("Failed to rename some attachments, old and new paths are saved into: %s", dumpPath)
  574. }()
  575. for _, attach := range attachments {
  576. if err = os.MkdirAll(path.Dir(attach.NewPath), os.ModePerm); err != nil {
  577. isSucceed = false
  578. return err
  579. }
  580. if err = os.Rename(attach.Path, attach.NewPath); err != nil {
  581. isSucceed = false
  582. return err
  583. }
  584. }
  585. return sess.Commit()
  586. }
  587. func renamePullRequestFields(x *xorm.Engine) (err error) {
  588. type PullRequest struct {
  589. ID int64 `xorm:"pk autoincr"`
  590. PullID int64 `xorm:"INDEX"`
  591. PullIndex int64
  592. HeadBarcnh string
  593. IssueID int64 `xorm:"INDEX"`
  594. Index int64
  595. HeadBranch string
  596. }
  597. if err = x.Sync(new(PullRequest)); err != nil {
  598. return fmt.Errorf("sync: %v", err)
  599. }
  600. results, err := x.Query("SELECT `id`,`pull_id`,`pull_index`,`head_barcnh` FROM `pull_request`")
  601. if err != nil {
  602. if strings.Contains(err.Error(), "no such column") {
  603. return nil
  604. }
  605. return fmt.Errorf("select pull requests: %v", err)
  606. }
  607. sess := x.NewSession()
  608. defer sess.Close()
  609. if err = sess.Begin(); err != nil {
  610. return err
  611. }
  612. var pull *PullRequest
  613. for _, pr := range results {
  614. pull = &PullRequest{
  615. ID: com.StrTo(pr["id"]).MustInt64(),
  616. IssueID: com.StrTo(pr["pull_id"]).MustInt64(),
  617. Index: com.StrTo(pr["pull_index"]).MustInt64(),
  618. HeadBranch: string(pr["head_barcnh"]),
  619. }
  620. if pull.Index == 0 {
  621. continue
  622. }
  623. if _, err = sess.ID(pull.ID).Update(pull); err != nil {
  624. return err
  625. }
  626. }
  627. return sess.Commit()
  628. }
  629. func cleanUpMigrateRepoInfo(x *xorm.Engine) (err error) {
  630. type (
  631. User struct {
  632. ID int64 `xorm:"pk autoincr"`
  633. LowerName string
  634. }
  635. Repository struct {
  636. ID int64 `xorm:"pk autoincr"`
  637. OwnerID int64
  638. LowerName string
  639. }
  640. )
  641. repos := make([]*Repository, 0, 25)
  642. if err = x.Where("is_mirror=?", false).Find(&repos); err != nil {
  643. return fmt.Errorf("select all non-mirror repositories: %v", err)
  644. }
  645. var user *User
  646. for _, repo := range repos {
  647. user = &User{ID: repo.OwnerID}
  648. has, err := x.Get(user)
  649. if err != nil {
  650. return fmt.Errorf("get owner of repository[%d - %d]: %v", repo.ID, repo.OwnerID, err)
  651. } else if !has {
  652. continue
  653. }
  654. configPath := filepath.Join(setting.RepoRootPath, user.LowerName, repo.LowerName+".git/config")
  655. // In case repository file is somehow missing.
  656. if !com.IsFile(configPath) {
  657. continue
  658. }
  659. cfg, err := ini.Load(configPath)
  660. if err != nil {
  661. return fmt.Errorf("open config file: %v", err)
  662. }
  663. cfg.DeleteSection("remote \"origin\"")
  664. if err = cfg.SaveToIndent(configPath, "\t"); err != nil {
  665. return fmt.Errorf("save config file: %v", err)
  666. }
  667. }
  668. return nil
  669. }
  670. func generateOrgRandsAndSalt(x *xorm.Engine) (err error) {
  671. type User struct {
  672. ID int64 `xorm:"pk autoincr"`
  673. Rands string `xorm:"VARCHAR(10)"`
  674. Salt string `xorm:"VARCHAR(10)"`
  675. }
  676. orgs := make([]*User, 0, 10)
  677. if err = x.Where("type=1").And("rands=''").Find(&orgs); err != nil {
  678. return fmt.Errorf("select all organizations: %v", err)
  679. }
  680. sess := x.NewSession()
  681. defer sess.Close()
  682. if err = sess.Begin(); err != nil {
  683. return err
  684. }
  685. for _, org := range orgs {
  686. if org.Rands, err = generate.GetRandomString(10); err != nil {
  687. return err
  688. }
  689. if org.Salt, err = generate.GetRandomString(10); err != nil {
  690. return err
  691. }
  692. if _, err = sess.ID(org.ID).Update(org); err != nil {
  693. return err
  694. }
  695. }
  696. return sess.Commit()
  697. }
  698. // TAction defines the struct for migrating table action
  699. type TAction struct {
  700. ID int64 `xorm:"pk autoincr"`
  701. CreatedUnix int64
  702. }
  703. // TableName will be invoked by XORM to customrize the table name
  704. func (t *TAction) TableName() string { return "action" }
  705. // TNotice defines the struct for migrating table notice
  706. type TNotice struct {
  707. ID int64 `xorm:"pk autoincr"`
  708. CreatedUnix int64
  709. }
  710. // TableName will be invoked by XORM to customrize the table name
  711. func (t *TNotice) TableName() string { return "notice" }
  712. // TComment defines the struct for migrating table comment
  713. type TComment struct {
  714. ID int64 `xorm:"pk autoincr"`
  715. CreatedUnix int64
  716. }
  717. // TableName will be invoked by XORM to customrize the table name
  718. func (t *TComment) TableName() string { return "comment" }
  719. // TIssue defines the struct for migrating table issue
  720. type TIssue struct {
  721. ID int64 `xorm:"pk autoincr"`
  722. DeadlineUnix int64
  723. CreatedUnix int64
  724. UpdatedUnix int64
  725. }
  726. // TableName will be invoked by XORM to customrize the table name
  727. func (t *TIssue) TableName() string { return "issue" }
  728. // TMilestone defines the struct for migrating table milestone
  729. type TMilestone struct {
  730. ID int64 `xorm:"pk autoincr"`
  731. DeadlineUnix int64
  732. ClosedDateUnix int64
  733. }
  734. // TableName will be invoked by XORM to customrize the table name
  735. func (t *TMilestone) TableName() string { return "milestone" }
  736. // TAttachment defines the struct for migrating table attachment
  737. type TAttachment struct {
  738. ID int64 `xorm:"pk autoincr"`
  739. CreatedUnix int64
  740. }
  741. // TableName will be invoked by XORM to customrize the table name
  742. func (t *TAttachment) TableName() string { return "attachment" }
  743. // TLoginSource defines the struct for migrating table login_source
  744. type TLoginSource struct {
  745. ID int64 `xorm:"pk autoincr"`
  746. CreatedUnix int64
  747. UpdatedUnix int64
  748. }
  749. // TableName will be invoked by XORM to customrize the table name
  750. func (t *TLoginSource) TableName() string { return "login_source" }
  751. // TPull defines the struct for migrating table pull_request
  752. type TPull struct {
  753. ID int64 `xorm:"pk autoincr"`
  754. MergedUnix int64
  755. }
  756. // TableName will be invoked by XORM to customrize the table name
  757. func (t *TPull) TableName() string { return "pull_request" }
  758. // TRelease defines the struct for migrating table release
  759. type TRelease struct {
  760. ID int64 `xorm:"pk autoincr"`
  761. CreatedUnix int64
  762. }
  763. // TableName will be invoked by XORM to customrize the table name
  764. func (t *TRelease) TableName() string { return "release" }
  765. // TRepo defines the struct for migrating table repository
  766. type TRepo struct {
  767. ID int64 `xorm:"pk autoincr"`
  768. CreatedUnix int64
  769. UpdatedUnix int64
  770. }
  771. // TableName will be invoked by XORM to customrize the table name
  772. func (t *TRepo) TableName() string { return "repository" }
  773. // TMirror defines the struct for migrating table mirror
  774. type TMirror struct {
  775. ID int64 `xorm:"pk autoincr"`
  776. UpdatedUnix int64
  777. NextUpdateUnix int64
  778. }
  779. // TableName will be invoked by XORM to customrize the table name
  780. func (t *TMirror) TableName() string { return "mirror" }
  781. // TPublicKey defines the struct for migrating table public_key
  782. type TPublicKey struct {
  783. ID int64 `xorm:"pk autoincr"`
  784. CreatedUnix int64
  785. UpdatedUnix int64
  786. }
  787. // TableName will be invoked by XORM to customrize the table name
  788. func (t *TPublicKey) TableName() string { return "public_key" }
  789. // TDeployKey defines the struct for migrating table deploy_key
  790. type TDeployKey struct {
  791. ID int64 `xorm:"pk autoincr"`
  792. CreatedUnix int64
  793. UpdatedUnix int64
  794. }
  795. // TableName will be invoked by XORM to customrize the table name
  796. func (t *TDeployKey) TableName() string { return "deploy_key" }
  797. // TAccessToken defines the struct for migrating table access_token
  798. type TAccessToken struct {
  799. ID int64 `xorm:"pk autoincr"`
  800. CreatedUnix int64
  801. UpdatedUnix int64
  802. }
  803. // TableName will be invoked by XORM to customrize the table name
  804. func (t *TAccessToken) TableName() string { return "access_token" }
  805. // TUser defines the struct for migrating table user
  806. type TUser struct {
  807. ID int64 `xorm:"pk autoincr"`
  808. CreatedUnix int64
  809. UpdatedUnix int64
  810. }
  811. // TableName will be invoked by XORM to customrize the table name
  812. func (t *TUser) TableName() string { return "user" }
  813. // TWebhook defines the struct for migrating table webhook
  814. type TWebhook struct {
  815. ID int64 `xorm:"pk autoincr"`
  816. CreatedUnix int64
  817. UpdatedUnix int64
  818. }
  819. // TableName will be invoked by XORM to customrize the table name
  820. func (t *TWebhook) TableName() string { return "webhook" }
  821. func convertDateToUnix(x *xorm.Engine) (err error) {
  822. log.Info("This migration could take up to minutes, please be patient.")
  823. type Bean struct {
  824. ID int64 `xorm:"pk autoincr"`
  825. Created time.Time
  826. Updated time.Time
  827. Merged time.Time
  828. Deadline time.Time
  829. ClosedDate time.Time
  830. NextUpdate time.Time
  831. }
  832. var tables = []struct {
  833. name string
  834. cols []string
  835. bean interface{}
  836. }{
  837. {"action", []string{"created"}, new(TAction)},
  838. {"notice", []string{"created"}, new(TNotice)},
  839. {"comment", []string{"created"}, new(TComment)},
  840. {"issue", []string{"deadline", "created", "updated"}, new(TIssue)},
  841. {"milestone", []string{"deadline", "closed_date"}, new(TMilestone)},
  842. {"attachment", []string{"created"}, new(TAttachment)},
  843. {"login_source", []string{"created", "updated"}, new(TLoginSource)},
  844. {"pull_request", []string{"merged"}, new(TPull)},
  845. {"release", []string{"created"}, new(TRelease)},
  846. {"repository", []string{"created", "updated"}, new(TRepo)},
  847. {"mirror", []string{"updated", "next_update"}, new(TMirror)},
  848. {"public_key", []string{"created", "updated"}, new(TPublicKey)},
  849. {"deploy_key", []string{"created", "updated"}, new(TDeployKey)},
  850. {"access_token", []string{"created", "updated"}, new(TAccessToken)},
  851. {"user", []string{"created", "updated"}, new(TUser)},
  852. {"webhook", []string{"created", "updated"}, new(TWebhook)},
  853. }
  854. for _, table := range tables {
  855. log.Info("Converting table: %s", table.name)
  856. if err = x.Sync2(table.bean); err != nil {
  857. return fmt.Errorf("Sync [table: %s]: %v", table.name, err)
  858. }
  859. offset := 0
  860. for {
  861. beans := make([]*Bean, 0, 100)
  862. if err = x.Table(table.name).Asc("id").Limit(100, offset).Find(&beans); err != nil {
  863. return fmt.Errorf("select beans [table: %s, offset: %d]: %v", table.name, offset, err)
  864. }
  865. log.Trace("Table [%s]: offset: %d, beans: %d", table.name, offset, len(beans))
  866. if len(beans) == 0 {
  867. break
  868. }
  869. offset += 100
  870. baseSQL := "UPDATE `" + table.name + "` SET "
  871. for _, bean := range beans {
  872. valSQLs := make([]string, 0, len(table.cols))
  873. for _, col := range table.cols {
  874. fieldSQL := ""
  875. fieldSQL += col + "_unix = "
  876. switch col {
  877. case "deadline":
  878. if bean.Deadline.IsZero() {
  879. continue
  880. }
  881. fieldSQL += com.ToStr(bean.Deadline.Unix())
  882. case "created":
  883. fieldSQL += com.ToStr(bean.Created.Unix())
  884. case "updated":
  885. fieldSQL += com.ToStr(bean.Updated.Unix())
  886. case "closed_date":
  887. fieldSQL += com.ToStr(bean.ClosedDate.Unix())
  888. case "merged":
  889. fieldSQL += com.ToStr(bean.Merged.Unix())
  890. case "next_update":
  891. fieldSQL += com.ToStr(bean.NextUpdate.Unix())
  892. }
  893. valSQLs = append(valSQLs, fieldSQL)
  894. }
  895. if len(valSQLs) == 0 {
  896. continue
  897. }
  898. if _, err = x.Exec(baseSQL + strings.Join(valSQLs, ",") + " WHERE id = " + com.ToStr(bean.ID)); err != nil {
  899. return fmt.Errorf("update bean [table: %s, id: %d]: %v", table.name, bean.ID, err)
  900. }
  901. }
  902. }
  903. }
  904. return nil
  905. }