diff options
Diffstat (limited to 'modules')
-rw-r--r-- | modules/hostmatcher/hostmatcher.go | 138 | ||||
-rw-r--r-- | modules/hostmatcher/hostmatcher_test.go | 79 | ||||
-rw-r--r-- | modules/hostmatcher/http.go | 58 | ||||
-rw-r--r-- | modules/lfs/client.go | 5 | ||||
-rw-r--r-- | modules/lfs/client_test.go | 4 | ||||
-rw-r--r-- | modules/lfs/http_client.go | 14 | ||||
-rw-r--r-- | modules/matchlist/matchlist.go | 46 | ||||
-rw-r--r-- | modules/repository/repo.go | 19 | ||||
-rw-r--r-- | modules/setting/migrations.go | 19 | ||||
-rw-r--r-- | modules/setting/webhook.go | 5 |
10 files changed, 247 insertions, 140 deletions
diff --git a/modules/hostmatcher/hostmatcher.go b/modules/hostmatcher/hostmatcher.go index f8a787c575..815b8c4e4a 100644 --- a/modules/hostmatcher/hostmatcher.go +++ b/modules/hostmatcher/hostmatcher.go @@ -13,15 +13,18 @@ import ( ) // HostMatchList is used to check if a host or IP is in a list. -// If you only need to do wildcard matching, consider to use modules/matchlist type HostMatchList struct { - hosts []string + SettingKeyHint string + SettingValue string + + // builtins networks + builtins []string + // patterns for host names (with wildcard support) + patterns []string + // ipNets is the CIDR network list ipNets []*net.IPNet } -// MatchBuiltinAll all hosts are matched -const MatchBuiltinAll = "*" - // MatchBuiltinExternal A valid non-private unicast IP, all hosts on public internet are matched const MatchBuiltinExternal = "external" @@ -31,9 +34,13 @@ const MatchBuiltinPrivate = "private" // MatchBuiltinLoopback 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included. const MatchBuiltinLoopback = "loopback" +func isBuiltin(s string) bool { + return s == MatchBuiltinExternal || s == MatchBuiltinPrivate || s == MatchBuiltinLoopback +} + // ParseHostMatchList parses the host list HostMatchList -func ParseHostMatchList(hostList string) *HostMatchList { - hl := &HostMatchList{} +func ParseHostMatchList(settingKeyHint string, hostList string) *HostMatchList { + hl := &HostMatchList{SettingKeyHint: settingKeyHint, SettingValue: hostList} for _, s := range strings.Split(hostList, ",") { s = strings.ToLower(strings.TrimSpace(s)) if s == "" { @@ -42,53 +49,106 @@ func ParseHostMatchList(hostList string) *HostMatchList { _, ipNet, err := net.ParseCIDR(s) if err == nil { hl.ipNets = append(hl.ipNets, ipNet) + } else if isBuiltin(s) { + hl.builtins = append(hl.builtins, s) } else { - hl.hosts = append(hl.hosts, s) + hl.patterns = append(hl.patterns, s) } } return hl } -// MatchesHostOrIP checks if the host or IP matches an allow/deny(block) list -func (hl *HostMatchList) MatchesHostOrIP(host string, ip net.IP) bool { - var matched bool - host = strings.ToLower(host) - ipStr := ip.String() -loop: - for _, hostInList := range hl.hosts { - switch hostInList { - case "": +// ParseSimpleMatchList parse a simple matchlist (no built-in networks, no CIDR support, only wildcard pattern match) +func ParseSimpleMatchList(settingKeyHint string, matchList string) *HostMatchList { + hl := &HostMatchList{ + SettingKeyHint: settingKeyHint, + SettingValue: matchList, + } + for _, s := range strings.Split(matchList, ",") { + s = strings.ToLower(strings.TrimSpace(s)) + if s == "" { continue - case MatchBuiltinAll: - matched = true - break loop + } + // we keep the same result as old `matchlist`, so no builtin/CIDR support here, we only match wildcard patterns + hl.patterns = append(hl.patterns, s) + } + return hl +} + +// AppendBuiltin appends more builtins to match +func (hl *HostMatchList) AppendBuiltin(builtin string) { + hl.builtins = append(hl.builtins, builtin) +} + +// IsEmpty checks if the checklist is empty +func (hl *HostMatchList) IsEmpty() bool { + return hl == nil || (len(hl.builtins) == 0 && len(hl.patterns) == 0 && len(hl.ipNets) == 0) +} + +func (hl *HostMatchList) checkPattern(host string) bool { + host = strings.ToLower(strings.TrimSpace(host)) + for _, pattern := range hl.patterns { + if matched, _ := filepath.Match(pattern, host); matched { + return true + } + } + return false +} + +func (hl *HostMatchList) checkIP(ip net.IP) bool { + for _, pattern := range hl.patterns { + if pattern == "*" { + return true + } + } + for _, builtin := range hl.builtins { + switch builtin { case MatchBuiltinExternal: - if matched = ip.IsGlobalUnicast() && !util.IsIPPrivate(ip); matched { - break loop + if ip.IsGlobalUnicast() && !util.IsIPPrivate(ip) { + return true } case MatchBuiltinPrivate: - if matched = util.IsIPPrivate(ip); matched { - break loop + if util.IsIPPrivate(ip) { + return true } case MatchBuiltinLoopback: - if matched = ip.IsLoopback(); matched { - break loop - } - default: - if matched, _ = filepath.Match(hostInList, host); matched { - break loop - } - if matched, _ = filepath.Match(hostInList, ipStr); matched { - break loop + if ip.IsLoopback() { + return true } } } - if !matched { - for _, ipNet := range hl.ipNets { - if matched = ipNet.Contains(ip); matched { - break - } + for _, ipNet := range hl.ipNets { + if ipNet.Contains(ip) { + return true } } - return matched + return false +} + +// MatchHostName checks if the host matches an allow/deny(block) list +func (hl *HostMatchList) MatchHostName(host string) bool { + if hl == nil { + return false + } + if hl.checkPattern(host) { + return true + } + if ip := net.ParseIP(host); ip != nil { + return hl.checkIP(ip) + } + return false +} + +// MatchIPAddr checks if the IP matches an allow/deny(block) list, it's safe to pass `nil` to `ip` +func (hl *HostMatchList) MatchIPAddr(ip net.IP) bool { + if hl == nil { + return false + } + host := ip.String() // nil-safe, we will get "<nil>" if ip is nil + return hl.checkPattern(host) || hl.checkIP(ip) +} + +// MatchHostOrIP checks if the host or IP matches an allow/deny(block) list +func (hl *HostMatchList) MatchHostOrIP(host string, ip net.IP) bool { + return hl.MatchHostName(host) || hl.MatchIPAddr(ip) } diff --git a/modules/hostmatcher/hostmatcher_test.go b/modules/hostmatcher/hostmatcher_test.go index 8eaafbdbc8..66030a32f1 100644 --- a/modules/hostmatcher/hostmatcher_test.go +++ b/modules/hostmatcher/hostmatcher_test.go @@ -20,17 +20,28 @@ func TestHostOrIPMatchesList(t *testing.T) { // for IPv6: "::1" is loopback, "fd00::/8" is private - hl := ParseHostMatchList("private, External, *.myDomain.com, 169.254.1.0/24") + hl := ParseHostMatchList("", "private, External, *.myDomain.com, 169.254.1.0/24") + + test := func(cases []tc) { + for _, c := range cases { + assert.Equalf(t, c.expected, hl.MatchHostOrIP(c.host, c.ip), "case domain=%s, ip=%v, expected=%v", c.host, c.ip, c.expected) + } + } + cases := []tc{ {"", net.IPv4zero, false}, {"", net.IPv6zero, false}, {"", net.ParseIP("127.0.0.1"), false}, + {"127.0.0.1", nil, false}, {"", net.ParseIP("::1"), false}, {"", net.ParseIP("10.0.1.1"), true}, + {"10.0.1.1", nil, true}, {"", net.ParseIP("192.168.1.1"), true}, + {"192.168.1.1", nil, true}, {"", net.ParseIP("fd00::1"), true}, + {"fd00::1", nil, true}, {"", net.ParseIP("8.8.8.8"), true}, {"", net.ParseIP("1001::1"), true}, @@ -39,13 +50,13 @@ func TestHostOrIPMatchesList(t *testing.T) { {"sub.mydomain.com", net.IPv4zero, true}, {"", net.ParseIP("169.254.1.1"), true}, + {"169.254.1.1", nil, true}, {"", net.ParseIP("169.254.2.2"), false}, + {"169.254.2.2", nil, false}, } - for _, c := range cases { - assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) - } + test(cases) - hl = ParseHostMatchList("loopback") + hl = ParseHostMatchList("", "loopback") cases = []tc{ {"", net.IPv4zero, false}, {"", net.ParseIP("127.0.0.1"), true}, @@ -59,11 +70,9 @@ func TestHostOrIPMatchesList(t *testing.T) { {"mydomain.com", net.IPv4zero, false}, } - for _, c := range cases { - assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) - } + test(cases) - hl = ParseHostMatchList("private") + hl = ParseHostMatchList("", "private") cases = []tc{ {"", net.IPv4zero, false}, {"", net.ParseIP("127.0.0.1"), false}, @@ -77,11 +86,9 @@ func TestHostOrIPMatchesList(t *testing.T) { {"mydomain.com", net.IPv4zero, false}, } - for _, c := range cases { - assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) - } + test(cases) - hl = ParseHostMatchList("external") + hl = ParseHostMatchList("", "external") cases = []tc{ {"", net.IPv4zero, false}, {"", net.ParseIP("127.0.0.1"), false}, @@ -95,11 +102,9 @@ func TestHostOrIPMatchesList(t *testing.T) { {"mydomain.com", net.IPv4zero, false}, } - for _, c := range cases { - assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) - } + test(cases) - hl = ParseHostMatchList("*") + hl = ParseHostMatchList("", "*") cases = []tc{ {"", net.IPv4zero, true}, {"", net.ParseIP("127.0.0.1"), true}, @@ -113,7 +118,43 @@ func TestHostOrIPMatchesList(t *testing.T) { {"mydomain.com", net.IPv4zero, true}, } - for _, c := range cases { - assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) + test(cases) + + // built-in network names can be escaped (warping the first char with `[]`) to be used as a real host name + // this mechanism is reversed for internal usage only (maybe for some rare cases), it's not supposed to be used by end users + // a real user should never use loopback/private/external as their host names + hl = ParseHostMatchList("", "loopback, [p]rivate") + cases = []tc{ + {"loopback", nil, false}, + {"", net.ParseIP("127.0.0.1"), true}, + {"private", nil, true}, + {"", net.ParseIP("192.168.1.1"), false}, + } + test(cases) + + hl = ParseSimpleMatchList("", "loopback, *.domain.com") + cases = []tc{ + {"loopback", nil, true}, + {"", net.ParseIP("127.0.0.1"), false}, + {"sub.domain.com", nil, true}, + {"other.com", nil, false}, + {"", net.ParseIP("1.1.1.1"), false}, + } + test(cases) + + hl = ParseSimpleMatchList("", "external") + cases = []tc{ + {"", net.ParseIP("192.168.1.1"), false}, + {"", net.ParseIP("1.1.1.1"), false}, + {"external", nil, true}, + } + test(cases) + + hl = ParseSimpleMatchList("", "") + cases = []tc{ + {"", net.ParseIP("192.168.1.1"), false}, + {"", net.ParseIP("1.1.1.1"), false}, + {"external", nil, false}, } + test(cases) } diff --git a/modules/hostmatcher/http.go b/modules/hostmatcher/http.go new file mode 100644 index 0000000000..9dae61a44c --- /dev/null +++ b/modules/hostmatcher/http.go @@ -0,0 +1,58 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package hostmatcher + +import ( + "context" + "fmt" + "net" + "syscall" + "time" +) + +// NewDialContext returns a DialContext for Transport, the DialContext will do allow/block list check +func NewDialContext(usage string, allowList *HostMatchList, blockList *HostMatchList) func(ctx context.Context, network, addr string) (net.Conn, error) { + // How Go HTTP Client works with redirection: + // transport.RoundTrip URL=http://domain.com, Host=domain.com + // transport.DialContext addrOrHost=domain.com:80 + // dialer.Control tcp4:11.22.33.44:80 + // transport.RoundTrip URL=http://www.domain.com/, Host=(empty here, in the direction, HTTP client doesn't fill the Host field) + // transport.DialContext addrOrHost=domain.com:80 + // dialer.Control tcp4:11.22.33.44:80 + return func(ctx context.Context, network, addrOrHost string) (net.Conn, error) { + dialer := net.Dialer{ + // default values comes from http.DefaultTransport + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + + Control: func(network, ipAddr string, c syscall.RawConn) (err error) { + var host string + if host, _, err = net.SplitHostPort(addrOrHost); err != nil { + return err + } + // in Control func, the addr was already resolved to IP:PORT format, there is no cost to do ResolveTCPAddr here + tcpAddr, err := net.ResolveTCPAddr(network, ipAddr) + if err != nil { + return fmt.Errorf("%s can only call HTTP servers via TCP, deny '%s(%s:%s)', err=%v", usage, host, network, ipAddr, err) + } + + var blockedError error + if blockList.MatchHostOrIP(host, tcpAddr.IP) { + blockedError = fmt.Errorf("%s can not call blocked HTTP servers (check your %s setting), deny '%s(%s)'", usage, blockList.SettingKeyHint, host, ipAddr) + } + + // if we have an allow-list, check the allow-list first + if !allowList.IsEmpty() { + if !allowList.MatchHostOrIP(host, tcpAddr.IP) { + return fmt.Errorf("%s can only call allowed HTTP servers (check your %s setting), deny '%s(%s)'", usage, allowList.SettingKeyHint, host, ipAddr) + } + } + // otherwise, we always follow the blocked list + return blockedError + }, + } + return dialer.DialContext(ctx, network, addrOrHost) + } +} diff --git a/modules/lfs/client.go b/modules/lfs/client.go index 81b047c5bd..aaf61aefcf 100644 --- a/modules/lfs/client.go +++ b/modules/lfs/client.go @@ -7,6 +7,7 @@ package lfs import ( "context" "io" + "net/http" "net/url" ) @@ -24,9 +25,9 @@ type Client interface { } // NewClient creates a LFS client -func NewClient(endpoint *url.URL, skipTLSVerify bool) Client { +func NewClient(endpoint *url.URL, httpTransport *http.Transport) Client { if endpoint.Scheme == "file" { return newFilesystemClient(endpoint) } - return newHTTPClient(endpoint, skipTLSVerify) + return newHTTPClient(endpoint, httpTransport) } diff --git a/modules/lfs/client_test.go b/modules/lfs/client_test.go index ee6b7a59fc..88986f06d6 100644 --- a/modules/lfs/client_test.go +++ b/modules/lfs/client_test.go @@ -13,10 +13,10 @@ import ( func TestNewClient(t *testing.T) { u, _ := url.Parse("file:///test") - c := NewClient(u, true) + c := NewClient(u, nil) assert.IsType(t, &FilesystemClient{}, c) u, _ = url.Parse("https://test.com/lfs") - c = NewClient(u, true) + c = NewClient(u, nil) assert.IsType(t, &HTTPClient{}, c) } diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go index 5df5ed33a9..a1a3e7f363 100644 --- a/modules/lfs/http_client.go +++ b/modules/lfs/http_client.go @@ -7,7 +7,6 @@ package lfs import ( "bytes" "context" - "crypto/tls" "errors" "fmt" "net/http" @@ -34,12 +33,15 @@ func (c *HTTPClient) BatchSize() int { return batchSize } -func newHTTPClient(endpoint *url.URL, skipTLSVerify bool) *HTTPClient { +func newHTTPClient(endpoint *url.URL, httpTransport *http.Transport) *HTTPClient { + if httpTransport == nil { + httpTransport = &http.Transport{ + Proxy: proxy.Proxy(), + } + } + hc := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: skipTLSVerify}, - Proxy: proxy.Proxy(), - }, + Transport: httpTransport, } client := &HTTPClient{ diff --git a/modules/matchlist/matchlist.go b/modules/matchlist/matchlist.go deleted file mode 100644 index b65ed909dc..0000000000 --- a/modules/matchlist/matchlist.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package matchlist - -import ( - "strings" - - "github.com/gobwas/glob" -) - -// Matchlist represents a block or allow list -type Matchlist struct { - ruleGlobs []glob.Glob -} - -// NewMatchlist creates a new block or allow list -func NewMatchlist(rules ...string) (*Matchlist, error) { - for i := range rules { - rules[i] = strings.ToLower(rules[i]) - } - list := Matchlist{ - ruleGlobs: make([]glob.Glob, 0, len(rules)), - } - - for _, rule := range rules { - rg, err := glob.Compile(rule) - if err != nil { - return nil, err - } - list.ruleGlobs = append(list.ruleGlobs, rg) - } - - return &list, nil -} - -// Match will matches -func (b *Matchlist) Match(u string) bool { - for _, r := range b.ruleGlobs { - if r.Match(u) { - return true - } - } - return false -} diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 5eec5a7314..dd54a99cc9 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -8,7 +8,7 @@ import ( "context" "fmt" "io" - "net/url" + "net/http" "path" "strings" "time" @@ -46,7 +46,10 @@ func WikiRemoteURL(remote string) string { } // MigrateRepositoryGitData starts migrating git related data after created migrating repository -func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models.Repository, opts migration.MigrateOptions) (*models.Repository, error) { +func MigrateRepositoryGitData(ctx context.Context, u *models.User, + repo *models.Repository, opts migration.MigrateOptions, + httpTransport *http.Transport, +) (*models.Repository, error) { repoPath := models.RepoPath(u.Name, opts.RepoName) if u.IsOrganization() { @@ -141,8 +144,9 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models. } if opts.LFS { - ep := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint) - if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, ep, setting.Migrations.SkipTLSVerify); err != nil { + endpoint := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint) + lfsClient := lfs.NewClient(endpoint, httpTransport) + if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, lfsClient); err != nil { log.Error("Failed to store missing LFS objects for repository: %v", err) } } @@ -336,8 +340,7 @@ func PushUpdateAddTag(repo *models.Repository, gitRepo *git.Repository, tagName } // StoreMissingLfsObjectsInRepository downloads missing LFS objects -func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, endpoint *url.URL, skipTLSVerify bool) error { - client := lfs.NewClient(endpoint, skipTLSVerify) +func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, lfsClient lfs.Client) error { contentStore := lfs.NewContentStore() pointerChan := make(chan lfs.PointerBlob) @@ -345,7 +348,7 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan) downloadObjects := func(pointers []lfs.Pointer) error { - err := client.Download(ctx, pointers, func(p lfs.Pointer, content io.ReadCloser, objectError error) error { + err := lfsClient.Download(ctx, pointers, func(p lfs.Pointer, content io.ReadCloser, objectError error) error { if objectError != nil { return objectError } @@ -411,7 +414,7 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi } batch = append(batch, pointerBlob.Pointer) - if len(batch) >= client.BatchSize() { + if len(batch) >= lfsClient.BatchSize() { if err := downloadObjects(batch); err != nil { return err } diff --git a/modules/setting/migrations.go b/modules/setting/migrations.go index b663b52f89..34d9037275 100644 --- a/modules/setting/migrations.go +++ b/modules/setting/migrations.go @@ -4,17 +4,13 @@ package setting -import ( - "strings" -) - var ( // Migrations settings Migrations = struct { MaxAttempts int RetryBackoff int - AllowedDomains []string - BlockedDomains []string + AllowedDomains string + BlockedDomains string AllowLocalNetworks bool SkipTLSVerify bool }{ @@ -28,15 +24,8 @@ func newMigrationsService() { Migrations.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Migrations.MaxAttempts) Migrations.RetryBackoff = sec.Key("RETRY_BACKOFF").MustInt(Migrations.RetryBackoff) - Migrations.AllowedDomains = sec.Key("ALLOWED_DOMAINS").Strings(",") - for i := range Migrations.AllowedDomains { - Migrations.AllowedDomains[i] = strings.ToLower(Migrations.AllowedDomains[i]) - } - Migrations.BlockedDomains = sec.Key("BLOCKED_DOMAINS").Strings(",") - for i := range Migrations.BlockedDomains { - Migrations.BlockedDomains[i] = strings.ToLower(Migrations.BlockedDomains[i]) - } - + Migrations.AllowedDomains = sec.Key("ALLOWED_DOMAINS").MustString("") + Migrations.BlockedDomains = sec.Key("BLOCKED_DOMAINS").MustString("") Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false) Migrations.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool(false) } diff --git a/modules/setting/webhook.go b/modules/setting/webhook.go index acd5bd0455..6284f397b1 100644 --- a/modules/setting/webhook.go +++ b/modules/setting/webhook.go @@ -7,7 +7,6 @@ package setting import ( "net/url" - "code.gitea.io/gitea/modules/hostmatcher" "code.gitea.io/gitea/modules/log" ) @@ -17,7 +16,7 @@ var ( QueueLength int DeliverTimeout int SkipTLSVerify bool - AllowedHostList *hostmatcher.HostMatchList + AllowedHostList string Types []string PagingNum int ProxyURL string @@ -38,7 +37,7 @@ func newWebhookService() { Webhook.QueueLength = sec.Key("QUEUE_LENGTH").MustInt(1000) Webhook.DeliverTimeout = sec.Key("DELIVER_TIMEOUT").MustInt(5) Webhook.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool() - Webhook.AllowedHostList = hostmatcher.ParseHostMatchList(sec.Key("ALLOWED_HOST_LIST").MustString(hostmatcher.MatchBuiltinExternal)) + Webhook.AllowedHostList = sec.Key("ALLOWED_HOST_LIST").MustString("") Webhook.Types = []string{"gitea", "gogs", "slack", "discord", "dingtalk", "telegram", "msteams", "feishu", "matrix", "wechatwork"} Webhook.PagingNum = sec.Key("PAGING_NUM").MustInt(10) Webhook.ProxyURL = sec.Key("PROXY_URL").MustString("") |