diff options
| -rw-r--r-- | contrib/environment-to-ini/environment-to-ini.go | 120 | ||||
| -rw-r--r-- | docs/content/doc/installation/with-docker-rootless.en-us.md | 15 | ||||
| -rw-r--r-- | docs/content/doc/installation/with-docker.en-us.md | 15 | ||||
| -rw-r--r-- | modules/setting/config_env.go | 142 | ||||
| -rw-r--r-- | modules/setting/config_env_test.go | 97 |
5 files changed, 278 insertions, 111 deletions
diff --git a/contrib/environment-to-ini/environment-to-ini.go b/contrib/environment-to-ini/environment-to-ini.go index b502c15cec..ae8535d891 100644 --- a/contrib/environment-to-ini/environment-to-ini.go +++ b/contrib/environment-to-ini/environment-to-ini.go @@ -5,8 +5,6 @@ package main import ( "os" - "regexp" - "strconv" "strings" "code.gitea.io/gitea/modules/log" @@ -14,7 +12,7 @@ import ( "code.gitea.io/gitea/modules/util" "github.com/urfave/cli" - ini "gopkg.in/ini.v1" + "gopkg.in/ini.v1" ) // EnvironmentPrefix environment variables prefixed with this represent ini values to write @@ -32,6 +30,10 @@ func main() { will be mapped to the ini section "[section_name]" and the key "KEY_NAME" with the value as provided. + Environment variables of the form "GITEA__SECTION_NAME__KEY_NAME__FILE" + will be mapped to the ini section "[section_name]" and the key + "KEY_NAME" with the value loaded from the specified file. + Environment variables are usually restricted to a reduced character set "0-9A-Z_" - in order to allow the setting of sections with characters outside of that set, they should be escaped as following: @@ -96,11 +98,11 @@ func runEnvironmentToIni(c *cli.Context) error { setting.SetCustomPathAndConf(providedCustom, providedConf, providedWorkPath) cfg := ini.Empty() - isFile, err := util.IsFile(setting.CustomConf) + confFileExists, err := util.IsFile(setting.CustomConf) if err != nil { log.Fatal("Unable to check if %s is a file. Error: %v", setting.CustomConf, err) } - if isFile { + if confFileExists { if err := cfg.Append(setting.CustomConf); err != nil { log.Fatal("Failed to load custom conf '%s': %v", setting.CustomConf, err) } @@ -109,47 +111,11 @@ func runEnvironmentToIni(c *cli.Context) error { } cfg.NameMapper = ini.SnackCase - changed := false + prefixGitea := c.String("prefix") + "__" + suffixFile := "__FILE" + changed := setting.EnvironmentToConfig(cfg, prefixGitea, suffixFile, os.Environ()) - prefix := c.String("prefix") + "__" - - for _, kv := range os.Environ() { - idx := strings.IndexByte(kv, '=') - if idx < 0 { - continue - } - eKey := kv[:idx] - value := kv[idx+1:] - if !strings.HasPrefix(eKey, prefix) { - continue - } - eKey = eKey[len(prefix):] - sectionName, keyName := DecodeSectionKey(eKey) - if len(keyName) == 0 { - continue - } - section, err := cfg.GetSection(sectionName) - if err != nil { - section, err = cfg.NewSection(sectionName) - if err != nil { - log.Error("Error creating section: %s : %v", sectionName, err) - continue - } - } - key := section.Key(keyName) - if key == nil { - key, err = section.NewKey(keyName, value) - if err != nil { - log.Error("Error creating key: %s in section: %s with value: %s : %v", keyName, sectionName, value, err) - continue - } - } - oldValue := key.Value() - if !changed && oldValue != value { - changed = true - } - key.SetValue(value) - } + // try to save the config file destination := c.String("out") if len(destination) == 0 { destination = setting.CustomConf @@ -161,6 +127,8 @@ func runEnvironmentToIni(c *cli.Context) error { return err } } + + // clear Gitea's specific environment variables if requested if c.Bool("clear") { for _, kv := range os.Environ() { idx := strings.IndexByte(kv, '=') @@ -168,69 +136,11 @@ func runEnvironmentToIni(c *cli.Context) error { continue } eKey := kv[:idx] - if strings.HasPrefix(eKey, prefix) { + if strings.HasPrefix(eKey, prefixGitea) { _ = os.Unsetenv(eKey) } } } - return nil -} -const escapeRegexpString = "_0[xX](([0-9a-fA-F][0-9a-fA-F])+)_" - -var escapeRegex = regexp.MustCompile(escapeRegexpString) - -// DecodeSectionKey will decode a portable string encoded Section__Key pair -// Portable strings are considered to be of the form [A-Z0-9_]* -// We will encode a disallowed value as the UTF8 byte string preceded by _0X and -// followed by _. E.g. _0X2C_ for a '-' and _0X2E_ for '.' -// Section and Key are separated by a plain '__'. -// The entire section can be encoded as a UTF8 byte string -func DecodeSectionKey(encoded string) (string, string) { - section := "" - key := "" - - inKey := false - last := 0 - escapeStringIndices := escapeRegex.FindAllStringIndex(encoded, -1) - for _, unescapeIdx := range escapeStringIndices { - preceding := encoded[last:unescapeIdx[0]] - if !inKey { - if splitter := strings.Index(preceding, "__"); splitter > -1 { - section += preceding[:splitter] - inKey = true - key += preceding[splitter+2:] - } else { - section += preceding - } - } else { - key += preceding - } - toDecode := encoded[unescapeIdx[0]+3 : unescapeIdx[1]-1] - decodedBytes := make([]byte, len(toDecode)/2) - for i := 0; i < len(toDecode)/2; i++ { - // Can ignore error here as we know these should be hexadecimal from the regexp - byteInt, _ := strconv.ParseInt(toDecode[2*i:2*i+2], 16, 0) - decodedBytes[i] = byte(byteInt) - } - if inKey { - key += string(decodedBytes) - } else { - section += string(decodedBytes) - } - last = unescapeIdx[1] - } - remaining := encoded[last:] - if !inKey { - if splitter := strings.Index(remaining, "__"); splitter > -1 { - section += remaining[:splitter] - key += remaining[splitter+2:] - } else { - section += remaining - } - } else { - key += remaining - } - section = strings.ToLower(section) - return section, key + return nil } diff --git a/docs/content/doc/installation/with-docker-rootless.en-us.md b/docs/content/doc/installation/with-docker-rootless.en-us.md index e1073a1d67..b8c76438f9 100644 --- a/docs/content/doc/installation/with-docker-rootless.en-us.md +++ b/docs/content/doc/installation/with-docker-rootless.en-us.md @@ -286,9 +286,18 @@ docker-compose up -d ## Managing Deployments With Environment Variables -In addition to the environment variables above, any settings in `app.ini` can be set or overridden with an environment variable of the form: `GITEA__SECTION_NAME__KEY_NAME`. These settings are applied each time the docker container starts. Full information [here](https://github.com/go-gitea/gitea/tree/main/contrib/environment-to-ini). - -These environment variables can be passed to the docker container in `docker-compose.yml`. The following example will enable an smtp mail server if the required env variables `GITEA__mailer__FROM`, `GITEA__mailer__HOST`, `GITEA__mailer__PASSWD` are set on the host or in a `.env` file in the same directory as `docker-compose.yml`: +In addition to the environment variables above, any settings in `app.ini` can be set +or overridden with an environment variable of the form: `GITEA__SECTION_NAME__KEY_NAME`. +These settings are applied each time the docker container starts. +Full information [here](https://github.com/go-gitea/gitea/tree/main/contrib/environment-to-ini). + +These environment variables can be passed to the docker container in `docker-compose.yml`. +The following example will enable a smtp mail server if the required env variables +`GITEA__mailer__FROM`, `GITEA__mailer__HOST`, `GITEA__mailer__PASSWD` are set on the host +or in a `.env` file in the same directory as `docker-compose.yml`. + +The settings can be also set or overridden with the content of a file by defining an environment variable of the form: +`GITEA__section_name__KEY_NAME__FILE` that points to a file. ```bash ... diff --git a/docs/content/doc/installation/with-docker.en-us.md b/docs/content/doc/installation/with-docker.en-us.md index 043a10c08f..e70a6ab133 100644 --- a/docs/content/doc/installation/with-docker.en-us.md +++ b/docs/content/doc/installation/with-docker.en-us.md @@ -287,9 +287,18 @@ docker-compose up -d ## Managing Deployments With Environment Variables -In addition to the environment variables above, any settings in `app.ini` can be set or overridden with an environment variable of the form: `GITEA__SECTION_NAME__KEY_NAME`. These settings are applied each time the docker container starts. Full information [here](https://github.com/go-gitea/gitea/tree/master/contrib/environment-to-ini). - -These environment variables can be passed to the docker container in `docker-compose.yml`. The following example will enable an smtp mail server if the required env variables `GITEA__mailer__FROM`, `GITEA__mailer__HOST`, `GITEA__mailer__PASSWD` are set on the host or in a `.env` file in the same directory as `docker-compose.yml`: +In addition to the environment variables above, any settings in `app.ini` can be set +or overridden with an environment variable of the form: `GITEA__SECTION_NAME__KEY_NAME`. +These settings are applied each time the docker container starts. +Full information [here](https://github.com/go-gitea/gitea/tree/master/contrib/environment-to-ini). + +These environment variables can be passed to the docker container in `docker-compose.yml`. +The following example will enable an smtp mail server if the required env variables +`GITEA__mailer__FROM`, `GITEA__mailer__HOST`, `GITEA__mailer__PASSWD` are set on the host +or in a `.env` file in the same directory as `docker-compose.yml`. + +The settings can be also set or overridden with the content of a file by defining an environment variable of the form: +`GITEA__section_name__KEY_NAME__FILE` that points to a file. ```bash ... diff --git a/modules/setting/config_env.go b/modules/setting/config_env.go new file mode 100644 index 0000000000..dca9f2bb47 --- /dev/null +++ b/modules/setting/config_env.go @@ -0,0 +1,142 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "os" + "regexp" + "strconv" + "strings" + + "code.gitea.io/gitea/modules/log" + + "gopkg.in/ini.v1" +) + +const escapeRegexpString = "_0[xX](([0-9a-fA-F][0-9a-fA-F])+)_" + +var escapeRegex = regexp.MustCompile(escapeRegexpString) + +// decodeEnvSectionKey will decode a portable string encoded Section__Key pair +// Portable strings are considered to be of the form [A-Z0-9_]* +// We will encode a disallowed value as the UTF8 byte string preceded by _0X and +// followed by _. E.g. _0X2C_ for a '-' and _0X2E_ for '.' +// Section and Key are separated by a plain '__'. +// The entire section can be encoded as a UTF8 byte string +func decodeEnvSectionKey(encoded string) (ok bool, section, key string) { + inKey := false + last := 0 + escapeStringIndices := escapeRegex.FindAllStringIndex(encoded, -1) + for _, unescapeIdx := range escapeStringIndices { + preceding := encoded[last:unescapeIdx[0]] + if !inKey { + if splitter := strings.Index(preceding, "__"); splitter > -1 { + section += preceding[:splitter] + inKey = true + key += preceding[splitter+2:] + } else { + section += preceding + } + } else { + key += preceding + } + toDecode := encoded[unescapeIdx[0]+3 : unescapeIdx[1]-1] + decodedBytes := make([]byte, len(toDecode)/2) + for i := 0; i < len(toDecode)/2; i++ { + // Can ignore error here as we know these should be hexadecimal from the regexp + byteInt, _ := strconv.ParseInt(toDecode[2*i:2*i+2], 16, 0) + decodedBytes[i] = byte(byteInt) + } + if inKey { + key += string(decodedBytes) + } else { + section += string(decodedBytes) + } + last = unescapeIdx[1] + } + remaining := encoded[last:] + if !inKey { + if splitter := strings.Index(remaining, "__"); splitter > -1 { + section += remaining[:splitter] + key += remaining[splitter+2:] + } else { + section += remaining + } + } else { + key += remaining + } + section = strings.ToLower(section) + ok = section != "" && key != "" + if !ok { + section = "" + key = "" + } + return ok, section, key +} + +// decodeEnvironmentKey decode the environment key to section and key +// The environment key is in the form of GITEA__SECTION__KEY or GITEA__SECTION__KEY__FILE +func decodeEnvironmentKey(prefixGitea, suffixFile, envKey string) (ok bool, section, key string, useFileValue bool) { + if !strings.HasPrefix(envKey, prefixGitea) { + return false, "", "", false + } + if strings.HasSuffix(envKey, suffixFile) { + useFileValue = true + envKey = envKey[:len(envKey)-len(suffixFile)] + } + ok, section, key = decodeEnvSectionKey(envKey[len(prefixGitea):]) + return ok, section, key, useFileValue +} + +func EnvironmentToConfig(cfg *ini.File, prefixGitea, suffixFile string, envs []string) (changed bool) { + for _, kv := range envs { + idx := strings.IndexByte(kv, '=') + if idx < 0 { + continue + } + + // parse the environment variable to config section name and key name + envKey := kv[:idx] + envValue := kv[idx+1:] + ok, sectionName, keyName, useFileValue := decodeEnvironmentKey(prefixGitea, suffixFile, envKey) + if !ok { + continue + } + + // use environment value as config value, or read the file content as value if the key indicates a file + keyValue := envValue + if useFileValue { + fileContent, err := os.ReadFile(envValue) + if err != nil { + log.Error("Error reading file for %s : %v", envKey, envValue, err) + continue + } + keyValue = string(fileContent) + } + + // try to set the config value if necessary + section, err := cfg.GetSection(sectionName) + if err != nil { + section, err = cfg.NewSection(sectionName) + if err != nil { + log.Error("Error creating section: %s : %v", sectionName, err) + continue + } + } + key := section.Key(keyName) + if key == nil { + key, err = section.NewKey(keyName, keyValue) + if err != nil { + log.Error("Error creating key: %s in section: %s with value: %s : %v", keyName, sectionName, keyValue, err) + continue + } + } + oldValue := key.Value() + if !changed && oldValue != keyValue { + changed = true + } + key.SetValue(keyValue) + } + return changed +} diff --git a/modules/setting/config_env_test.go b/modules/setting/config_env_test.go new file mode 100644 index 0000000000..d49464ecf7 --- /dev/null +++ b/modules/setting/config_env_test.go @@ -0,0 +1,97 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "gopkg.in/ini.v1" +) + +func TestDecodeEnvSectionKey(t *testing.T) { + ok, section, key := decodeEnvSectionKey("SEC__KEY") + assert.True(t, ok) + assert.Equal(t, "sec", section) + assert.Equal(t, "KEY", key) + + ok, section, key = decodeEnvSectionKey("sec__key") + assert.True(t, ok) + assert.Equal(t, "sec", section) + assert.Equal(t, "key", key) + + ok, section, key = decodeEnvSectionKey("LOG_0x2E_CONSOLE__STDERR") + assert.True(t, ok) + assert.Equal(t, "log.console", section) + assert.Equal(t, "STDERR", key) + + ok, section, key = decodeEnvSectionKey("SEC") + assert.False(t, ok) + assert.Equal(t, "", section) + assert.Equal(t, "", key) +} + +func TestDecodeEnvironmentKey(t *testing.T) { + prefix := "GITEA__" + suffix := "__FILE" + + ok, section, key, file := decodeEnvironmentKey(prefix, suffix, "SEC__KEY") + assert.False(t, ok) + assert.Equal(t, "", section) + assert.Equal(t, "", key) + assert.False(t, file) + + ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC") + assert.False(t, ok) + assert.Equal(t, "", section) + assert.Equal(t, "", key) + assert.False(t, file) + + ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__KEY") + assert.True(t, ok) + assert.Equal(t, "sec", section) + assert.Equal(t, "KEY", key) + assert.False(t, file) + + // with "__FILE" suffix, it doesn't support to write "[sec].FILE" to config (no such key FILE is used in Gitea) + // but it could be fixed in the future by adding a new suffix like "__VALUE" (no such key VALUE is used in Gitea either) + ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__FILE") + assert.False(t, ok) + assert.Equal(t, "", section) + assert.Equal(t, "", key) + assert.True(t, file) + + ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__KEY__FILE") + assert.True(t, ok) + assert.Equal(t, "sec", section) + assert.Equal(t, "KEY", key) + assert.True(t, file) +} + +func TestEnvironmentToConfig(t *testing.T) { + cfg := ini.Empty() + + changed := EnvironmentToConfig(cfg, "GITEA__", "__FILE", nil) + assert.False(t, changed) + + cfg, err := ini.Load([]byte(` +[sec] +key = old +`)) + assert.NoError(t, err) + + changed = EnvironmentToConfig(cfg, "GITEA__", "__FILE", []string{"GITEA__sec__key=new"}) + assert.True(t, changed) + assert.Equal(t, "new", cfg.Section("sec").Key("key").String()) + + changed = EnvironmentToConfig(cfg, "GITEA__", "__FILE", []string{"GITEA__sec__key=new"}) + assert.False(t, changed) + + tmpFile := t.TempDir() + "/the-file" + _ = os.WriteFile(tmpFile, []byte("value-from-file"), 0o644) + changed = EnvironmentToConfig(cfg, "GITEA__", "__FILE", []string{"GITEA__sec__key__FILE=" + tmpFile}) + assert.True(t, changed) + assert.Equal(t, "value-from-file", cfg.Section("sec").Key("key").String()) +} |