diff options
author | wxiaoguang <wxiaoguang@gmail.com> | 2023-05-24 11:37:22 +0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-05-24 11:37:22 +0800 |
commit | c21605951b581440bb08b65d5907b1cd4e0ab6c5 (patch) | |
tree | 9ca38487c366009c65abecd5d94b7afbefdaf79b /modules/setting | |
parent | 1aa9107feaa3a04da2485d63ab7211a5cb6686f1 (diff) | |
download | gitea-c21605951b581440bb08b65d5907b1cd4e0ab6c5.tar.gz gitea-c21605951b581440bb08b65d5907b1cd4e0ab6c5.zip |
Make environment-to-ini support loading key value from file (#24832)
Replace #19857
Close #19856
Close #10311
Close #10123
Major changes:
1. Move a lot of code from `environment-to-ini.go` to `config_env.go` to
make them testable.
2. Add `__FILE` support
3. Update documents
4. Add tests
Diffstat (limited to 'modules/setting')
-rw-r--r-- | modules/setting/config_env.go | 142 | ||||
-rw-r--r-- | modules/setting/config_env_test.go | 97 |
2 files changed, 239 insertions, 0 deletions
diff --git a/modules/setting/config_env.go b/modules/setting/config_env.go new file mode 100644 index 0000000000..dca9f2bb47 --- /dev/null +++ b/modules/setting/config_env.go @@ -0,0 +1,142 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "os" + "regexp" + "strconv" + "strings" + + "code.gitea.io/gitea/modules/log" + + "gopkg.in/ini.v1" +) + +const escapeRegexpString = "_0[xX](([0-9a-fA-F][0-9a-fA-F])+)_" + +var escapeRegex = regexp.MustCompile(escapeRegexpString) + +// decodeEnvSectionKey will decode a portable string encoded Section__Key pair +// Portable strings are considered to be of the form [A-Z0-9_]* +// We will encode a disallowed value as the UTF8 byte string preceded by _0X and +// followed by _. E.g. _0X2C_ for a '-' and _0X2E_ for '.' +// Section and Key are separated by a plain '__'. +// The entire section can be encoded as a UTF8 byte string +func decodeEnvSectionKey(encoded string) (ok bool, section, key string) { + inKey := false + last := 0 + escapeStringIndices := escapeRegex.FindAllStringIndex(encoded, -1) + for _, unescapeIdx := range escapeStringIndices { + preceding := encoded[last:unescapeIdx[0]] + if !inKey { + if splitter := strings.Index(preceding, "__"); splitter > -1 { + section += preceding[:splitter] + inKey = true + key += preceding[splitter+2:] + } else { + section += preceding + } + } else { + key += preceding + } + toDecode := encoded[unescapeIdx[0]+3 : unescapeIdx[1]-1] + decodedBytes := make([]byte, len(toDecode)/2) + for i := 0; i < len(toDecode)/2; i++ { + // Can ignore error here as we know these should be hexadecimal from the regexp + byteInt, _ := strconv.ParseInt(toDecode[2*i:2*i+2], 16, 0) + decodedBytes[i] = byte(byteInt) + } + if inKey { + key += string(decodedBytes) + } else { + section += string(decodedBytes) + } + last = unescapeIdx[1] + } + remaining := encoded[last:] + if !inKey { + if splitter := strings.Index(remaining, "__"); splitter > -1 { + section += remaining[:splitter] + key += remaining[splitter+2:] + } else { + section += remaining + } + } else { + key += remaining + } + section = strings.ToLower(section) + ok = section != "" && key != "" + if !ok { + section = "" + key = "" + } + return ok, section, key +} + +// decodeEnvironmentKey decode the environment key to section and key +// The environment key is in the form of GITEA__SECTION__KEY or GITEA__SECTION__KEY__FILE +func decodeEnvironmentKey(prefixGitea, suffixFile, envKey string) (ok bool, section, key string, useFileValue bool) { + if !strings.HasPrefix(envKey, prefixGitea) { + return false, "", "", false + } + if strings.HasSuffix(envKey, suffixFile) { + useFileValue = true + envKey = envKey[:len(envKey)-len(suffixFile)] + } + ok, section, key = decodeEnvSectionKey(envKey[len(prefixGitea):]) + return ok, section, key, useFileValue +} + +func EnvironmentToConfig(cfg *ini.File, prefixGitea, suffixFile string, envs []string) (changed bool) { + for _, kv := range envs { + idx := strings.IndexByte(kv, '=') + if idx < 0 { + continue + } + + // parse the environment variable to config section name and key name + envKey := kv[:idx] + envValue := kv[idx+1:] + ok, sectionName, keyName, useFileValue := decodeEnvironmentKey(prefixGitea, suffixFile, envKey) + if !ok { + continue + } + + // use environment value as config value, or read the file content as value if the key indicates a file + keyValue := envValue + if useFileValue { + fileContent, err := os.ReadFile(envValue) + if err != nil { + log.Error("Error reading file for %s : %v", envKey, envValue, err) + continue + } + keyValue = string(fileContent) + } + + // try to set the config value if necessary + section, err := cfg.GetSection(sectionName) + if err != nil { + section, err = cfg.NewSection(sectionName) + if err != nil { + log.Error("Error creating section: %s : %v", sectionName, err) + continue + } + } + key := section.Key(keyName) + if key == nil { + key, err = section.NewKey(keyName, keyValue) + if err != nil { + log.Error("Error creating key: %s in section: %s with value: %s : %v", keyName, sectionName, keyValue, err) + continue + } + } + oldValue := key.Value() + if !changed && oldValue != keyValue { + changed = true + } + key.SetValue(keyValue) + } + return changed +} diff --git a/modules/setting/config_env_test.go b/modules/setting/config_env_test.go new file mode 100644 index 0000000000..d49464ecf7 --- /dev/null +++ b/modules/setting/config_env_test.go @@ -0,0 +1,97 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "gopkg.in/ini.v1" +) + +func TestDecodeEnvSectionKey(t *testing.T) { + ok, section, key := decodeEnvSectionKey("SEC__KEY") + assert.True(t, ok) + assert.Equal(t, "sec", section) + assert.Equal(t, "KEY", key) + + ok, section, key = decodeEnvSectionKey("sec__key") + assert.True(t, ok) + assert.Equal(t, "sec", section) + assert.Equal(t, "key", key) + + ok, section, key = decodeEnvSectionKey("LOG_0x2E_CONSOLE__STDERR") + assert.True(t, ok) + assert.Equal(t, "log.console", section) + assert.Equal(t, "STDERR", key) + + ok, section, key = decodeEnvSectionKey("SEC") + assert.False(t, ok) + assert.Equal(t, "", section) + assert.Equal(t, "", key) +} + +func TestDecodeEnvironmentKey(t *testing.T) { + prefix := "GITEA__" + suffix := "__FILE" + + ok, section, key, file := decodeEnvironmentKey(prefix, suffix, "SEC__KEY") + assert.False(t, ok) + assert.Equal(t, "", section) + assert.Equal(t, "", key) + assert.False(t, file) + + ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC") + assert.False(t, ok) + assert.Equal(t, "", section) + assert.Equal(t, "", key) + assert.False(t, file) + + ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__KEY") + assert.True(t, ok) + assert.Equal(t, "sec", section) + assert.Equal(t, "KEY", key) + assert.False(t, file) + + // with "__FILE" suffix, it doesn't support to write "[sec].FILE" to config (no such key FILE is used in Gitea) + // but it could be fixed in the future by adding a new suffix like "__VALUE" (no such key VALUE is used in Gitea either) + ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__FILE") + assert.False(t, ok) + assert.Equal(t, "", section) + assert.Equal(t, "", key) + assert.True(t, file) + + ok, section, key, file = decodeEnvironmentKey(prefix, suffix, "GITEA__SEC__KEY__FILE") + assert.True(t, ok) + assert.Equal(t, "sec", section) + assert.Equal(t, "KEY", key) + assert.True(t, file) +} + +func TestEnvironmentToConfig(t *testing.T) { + cfg := ini.Empty() + + changed := EnvironmentToConfig(cfg, "GITEA__", "__FILE", nil) + assert.False(t, changed) + + cfg, err := ini.Load([]byte(` +[sec] +key = old +`)) + assert.NoError(t, err) + + changed = EnvironmentToConfig(cfg, "GITEA__", "__FILE", []string{"GITEA__sec__key=new"}) + assert.True(t, changed) + assert.Equal(t, "new", cfg.Section("sec").Key("key").String()) + + changed = EnvironmentToConfig(cfg, "GITEA__", "__FILE", []string{"GITEA__sec__key=new"}) + assert.False(t, changed) + + tmpFile := t.TempDir() + "/the-file" + _ = os.WriteFile(tmpFile, []byte("value-from-file"), 0o644) + changed = EnvironmentToConfig(cfg, "GITEA__", "__FILE", []string{"GITEA__sec__key__FILE=" + tmpFile}) + assert.True(t, changed) + assert.Equal(t, "value-from-file", cfg.Section("sec").Key("key").String()) +} |