Improve install code to avoid low-level mistakes. (#17779)

* Improve install code to avoid low-level mistakes.

If a user tries to do a re-install in a Gitea database, they gets a warning and double check.
When Gitea runs, it never create empty app.ini automatically.

Also some small (related) refactoring:

* Refactor db.InitEngine related logic make it more clean (especially for the install code)
* Move some i18n strings out from setting.go to make the setting.go can be easily maintained.
* Show errors in CLI code if an incorrect app.ini is used.
* APP_DATA_PATH is created when installing, and checked when starting (no empty directory is created any more).

6 files changed, 144 insertions, 50 deletions

diff --git a/modules/doctor/doctor.go b/modules/doctor/doctor.go
index 6451788f9d..20a32f1865 100644
--- a/modules/doctor/doctor.go
+++ b/modules/doctor/doctor.go
@@ -44,7 +44,7 @@ func (w *wrappedLevelLogger) Log(skip int, level log.Level, format string, v ...
 }
 
 func initDBDisableConsole(ctx context.Context, disableConsole bool) error {
-	setting.NewContext()
+	setting.LoadFromExisting()
 	setting.InitDBConfig()
 
 	setting.NewXORMLogService(disableConsole)
diff --git a/modules/doctor/paths.go b/modules/doctor/paths.go
index 88172d3150..b4eab631ba 100644
--- a/modules/doctor/paths.go
+++ b/modules/doctor/paths.go
@@ -67,7 +67,7 @@ func checkConfigurationFiles(logger log.Logger, autofix bool) error {
 		return err
 	}
 
-	setting.NewContext()
+	setting.LoadFromExisting()
 
 	configurationFiles := []configurationFile{
 		{"Configuration File Path", setting.CustomConf, false, true, false},
diff --git a/modules/private/internal.go b/modules/private/internal.go
index f5b5db0ca1..0a39ca7b8e 100644
--- a/modules/private/internal.go
+++ b/modules/private/internal.go
@@ -13,14 +13,18 @@ import (
 
 	"code.gitea.io/gitea/modules/httplib"
 	"code.gitea.io/gitea/modules/json"
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 )
 
 func newRequest(ctx context.Context, url, method string) *httplib.Request {
+	if setting.InternalToken == "" {
+		log.Fatal(`The INTERNAL_TOKEN setting is missing from the configuration file: %q.
+Ensure you are running in the correct environment or set the correct configuration file with -c.`, setting.CustomConf)
+	}
 	return httplib.NewRequest(url, method).
 		SetContext(ctx).
-		Header("Authorization",
-			fmt.Sprintf("Bearer %s", setting.InternalToken))
+		Header("Authorization", fmt.Sprintf("Bearer %s", setting.InternalToken))
 }
 
 // Response internal request response
@@ -44,9 +48,6 @@ func newInternalRequest(ctx context.Context, url, method string) *httplib.Reques
 	})
 	if setting.Protocol == setting.UnixSocket {
 		req.SetTransport(&http.Transport{
-			Dial: func(_, _ string) (net.Conn, error) {
-				return net.Dial("unix", setting.HTTPAddr)
-			},
 			DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
 				var d net.Dialer
 				return d.DialContext(ctx, "unix", setting.HTTPAddr)
diff --git a/modules/setting/directory.go b/modules/setting/directory.go
new file mode 100644
index 0000000000..5dcdd89c04
--- /dev/null
+++ b/modules/setting/directory.go
@@ -0,0 +1,40 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package setting
+
+import (
+	"fmt"
+	"os"
+)
+
+// PrepareAppDataPath creates app data directory if necessary
+func PrepareAppDataPath() error {
+	// FIXME: There are too many calls to MkdirAll in old code. It is incorrect.
+	// For example, if someDir=/mnt/vol1/gitea-home/data, if the mount point /mnt/vol1 is not mounted when Gitea runs,
+	// then gitea will make new empty directories in /mnt/vol1, all are stored in the root filesystem.
+	// The correct behavior should be: creating parent directories is end users' duty. We only create sub-directories in existing parent directories.
+	// For quickstart, the parent directories should be created automatically for first startup (eg: a flag or a check of INSTALL_LOCK).
+	// Now we can take the first step to do correctly (using Mkdir) in other packages, and prepare the AppDataPath here, then make a refactor in future.
+
+	st, err := os.Stat(AppDataPath)
+
+	if os.IsNotExist(err) {
+		err = os.MkdirAll(AppDataPath, os.ModePerm)
+		if err != nil {
+			return fmt.Errorf("unable to create the APP_DATA_PATH directory: %q, Error: %v", AppDataPath, err)
+		}
+		return nil
+	}
+
+	if err != nil {
+		return fmt.Errorf("unable to use APP_DATA_PATH %q. Error: %v", AppDataPath, err)
+	}
+
+	if !st.IsDir() /* also works for symlink */ {
+		return fmt.Errorf("the APP_DATA_PATH %q is not a directory (or symlink to a directory) and can't be used", AppDataPath)
+	}
+
+	return nil
+}
diff --git a/modules/setting/i18n.go b/modules/setting/i18n.go
new file mode 100644
index 0000000000..113e1b5585
--- /dev/null
+++ b/modules/setting/i18n.go
@@ -0,0 +1,51 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package setting
+
+// defaultI18nLangNames must be a slice, we need the order
+var defaultI18nLangNames = []string{
+	"en-US", "English",
+	"zh-CN", "简体中文",
+	"zh-HK", "繁體中文（香港）",
+	"zh-TW", "繁體中文（台灣）",
+	"de-DE", "Deutsch",
+	"fr-FR", "français",
+	"nl-NL", "Nederlands",
+	"lv-LV", "latviešu",
+	"ru-RU", "русский",
+	"uk-UA", "Українська",
+	"ja-JP", "日本語",
+	"es-ES", "español",
+	"pt-BR", "português do Brasil",
+	"pt-PT", "Português de Portugal",
+	"pl-PL", "polski",
+	"bg-BG", "български",
+	"it-IT", "italiano",
+	"fi-FI", "suomi",
+	"tr-TR", "Türkçe",
+	"cs-CZ", "čeština",
+	"sr-SP", "српски",
+	"sv-SE", "svenska",
+	"ko-KR", "한국어",
+	"el-GR", "ελληνικά",
+	"fa-IR", "فارسی",
+	"hu-HU", "magyar nyelv",
+	"id-ID", "bahasa Indonesia",
+	"ml-IN", "മലയാളം",
+}
+
+func defaultI18nLangs() (res []string) {
+	for i := 0; i < len(defaultI18nLangNames); i += 2 {
+		res = append(res, defaultI18nLangNames[i])
+	}
+	return
+}
+
+func defaultI18nNames() (res []string) {
+	for i := 0; i < len(defaultI18nLangNames); i += 2 {
+		res = append(res, defaultI18nLangNames[i+1])
+	}
+	return
+}
diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index 15cdc1fe3a..d219dbaafd 100644
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -546,9 +546,27 @@ func SetCustomPathAndConf(providedCustom, providedConf, providedWorkPath string)
 	}
 }
 
-// NewContext initializes configuration context.
+// LoadFromExisting initializes setting options from an existing config file (app.ini)
+func LoadFromExisting() {
+	loadFromConf(false)
+}
+
+// LoadAllowEmpty initializes setting options, it's also fine that if the config file (app.ini) doesn't exist
+func LoadAllowEmpty() {
+	loadFromConf(true)
+}
+
+// LoadForTest initializes setting options for tests
+func LoadForTest() {
+	loadFromConf(true)
+	if err := PrepareAppDataPath(); err != nil {
+		log.Fatal("Can not prepare APP_DATA_PATH: %v", err)
+	}
+}
+
+// loadFromConf initializes configuration context.
 // NOTE: do not print any log except error.
-func NewContext() {
+func loadFromConf(allowEmpty bool) {
 	Cfg = ini.Empty()
 
 	if WritePIDFile && len(PIDFile) > 0 {
@@ -563,9 +581,10 @@ func NewContext() {
 		if err := Cfg.Append(CustomConf); err != nil {
 			log.Fatal("Failed to load custom conf '%s': %v", CustomConf, err)
 		}
-	} else {
-		log.Warn("Custom config '%s' not found, ignore this if you're running first time", CustomConf)
-	}
+	} else if !allowEmpty {
+		log.Fatal("Unable to find configuration file: %q.\nEnsure you are running in the correct environment or set the correct configuration file with -c.", CustomConf)
+	} // else: no config file, a config file might be created at CustomConf later (might not)
+
 	Cfg.NameMapper = ini.SnackCase
 
 	homeDir, err := com.HomeDir()
@@ -698,18 +717,7 @@ func NewContext() {
 	StaticRootPath = sec.Key("STATIC_ROOT_PATH").MustString(StaticRootPath)
 	StaticCacheTime = sec.Key("STATIC_CACHE_TIME").MustDuration(6 * time.Hour)
 	AppDataPath = sec.Key("APP_DATA_PATH").MustString(path.Join(AppWorkPath, "data"))
-	if _, err = os.Stat(AppDataPath); err != nil {
-		// FIXME: There are too many calls to MkdirAll in old code. It is incorrect.
-		// For example, if someDir=/mnt/vol1/gitea-home/data, if the mount point /mnt/vol1 is not mounted when Gitea runs,
-		// then gitea will make new empty directories in /mnt/vol1, all are stored in the root filesystem.
-		// The correct behavior should be: creating parent directories is end users' duty. We only create sub-directories in existing parent directories.
-		// For quickstart, the parent directories should be created automatically for first startup (eg: a flag or a check of INSTALL_LOCK).
-		// Now we can take the first step to do correctly (using Mkdir) in other packages, and prepare the AppDataPath here, then make a refactor in future.
-		err = os.MkdirAll(AppDataPath, os.ModePerm)
-		if err != nil {
-			log.Fatal("Failed to create the directory for app data path '%s'", AppDataPath)
-		}
-	}
+
 	EnableGzip = sec.Key("ENABLE_GZIP").MustBool()
 	EnablePprof = sec.Key("ENABLE_PPROF").MustBool(false)
 	PprofDataPath = sec.Key("PPROF_DATA_PATH").MustString(path.Join(AppWorkPath, "data/tmp/pprof"))
@@ -864,6 +872,10 @@ func NewContext() {
 	SuccessfulTokensCacheSize = sec.Key("SUCCESSFUL_TOKENS_CACHE_SIZE").MustInt(20)
 
 	InternalToken = loadInternalToken(sec)
+	if InstallLock && InternalToken == "" {
+		// if Gitea has been installed but the InternalToken hasn't been generated (upgrade from an old release), we should generate
+		generateSaveInternalToken()
+	}
 
 	cfgdata := sec.Key("PASSWORD_COMPLEXITY").Strings(",")
 	if len(cfgdata) == 0 {
@@ -975,19 +987,11 @@ func NewContext() {
 
 	Langs = Cfg.Section("i18n").Key("LANGS").Strings(",")
 	if len(Langs) == 0 {
-		Langs = []string{
-			"en-US", "zh-CN", "zh-HK", "zh-TW", "de-DE", "fr-FR", "nl-NL", "lv-LV",
-			"ru-RU", "uk-UA", "ja-JP", "es-ES", "pt-BR", "pt-PT", "pl-PL", "bg-BG",
-			"it-IT", "fi-FI", "tr-TR", "cs-CZ", "sr-SP", "sv-SE", "ko-KR", "el-GR",
-			"fa-IR", "hu-HU", "id-ID", "ml-IN"}
+		Langs = defaultI18nLangs()
 	}
 	Names = Cfg.Section("i18n").Key("NAMES").Strings(",")
 	if len(Names) == 0 {
-		Names = []string{"English", "简体中文", "繁體中文（香港）", "繁體中文（台灣）", "Deutsch",
-			"français", "Nederlands", "latviešu", "русский", "Українська", "日本語",
-			"español", "português do Brasil", "Português de Portugal", "polski", "български",
-			"italiano", "suomi", "Türkçe", "čeština", "српски", "svenska", "한국어", "ελληνικά",
-			"فارسی", "magyar nyelv", "bahasa Indonesia", "മലയാളം"}
+		Names = defaultI18nNames()
 	}
 
 	ShowFooterBranding = Cfg.Section("other").Key("SHOW_FOOTER_BRANDING").MustBool(false)
@@ -1054,8 +1058,8 @@ func parseAuthorizedPrincipalsAllow(values []string) ([]string, bool) {
 
 func loadInternalToken(sec *ini.Section) string {
 	uri := sec.Key("INTERNAL_TOKEN_URI").String()
-	if len(uri) == 0 {
-		return loadOrGenerateInternalToken(sec)
+	if uri == "" {
+		return sec.Key("INTERNAL_TOKEN").String()
 	}
 	tempURI, err := url.Parse(uri)
 	if err != nil {
@@ -1092,21 +1096,17 @@ func loadInternalToken(sec *ini.Section) string {
 	return ""
 }
 
-func loadOrGenerateInternalToken(sec *ini.Section) string {
-	var err error
-	token := sec.Key("INTERNAL_TOKEN").String()
-	if len(token) == 0 {
-		token, err = generate.NewInternalToken()
-		if err != nil {
-			log.Fatal("Error generate internal token: %v", err)
-		}
-
-		// Save secret
-		CreateOrAppendToCustomConf(func(cfg *ini.File) {
-			cfg.Section("security").Key("INTERNAL_TOKEN").SetValue(token)
-		})
+// generateSaveInternalToken generates and saves the internal token to app.ini
+func generateSaveInternalToken() {
+	token, err := generate.NewInternalToken()
+	if err != nil {
+		log.Fatal("Error generate internal token: %v", err)
 	}
-	return token
+
+	InternalToken = token
+	CreateOrAppendToCustomConf(func(cfg *ini.File) {
+		cfg.Section("security").Key("INTERNAL_TOKEN").SetValue(token)
+	})
 }
 
 // MakeAbsoluteAssetURL returns the absolute asset url prefix without a trailing slash
@@ -1186,6 +1186,8 @@ func CreateOrAppendToCustomConf(callback func(cfg *ini.File)) {
 
 	callback(cfg)
 
+	log.Info("Settings saved to: %q", CustomConf)
+
 	if err := os.MkdirAll(filepath.Dir(CustomConf), os.ModePerm); err != nil {
 		log.Fatal("failed to create '%s': %v", CustomConf, err)
 		return