Use hostmacher to replace matchlist. And we introduce a better DialContext to do a full host/IP check, otherwise the attackers can still bypass the allow/block list by a 302 redirection.tags/v1.16.0-rc1
@@ -2114,7 +2114,7 @@ PATH = | |||
;ALLOWED_DOMAINS = | |||
;; | |||
;; Blocklist for migrating, default is blank. Multiple domains could be separated by commas. | |||
;; When ALLOWED_DOMAINS is not blank, this option will be ignored. | |||
;; When ALLOWED_DOMAINS is not blank, this option has a higher priority to deny domains. | |||
;BLOCKED_DOMAINS = | |||
;; | |||
;; Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291 (false by default) |
@@ -1045,7 +1045,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf | |||
- `MAX_ATTEMPTS`: **3**: Max attempts per http/https request on migrations. | |||
- `RETRY_BACKOFF`: **3**: Backoff time per http/https request retry (seconds) | |||
- `ALLOWED_DOMAINS`: **\<empty\>**: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas. | |||
- `BLOCKED_DOMAINS`: **\<empty\>**: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWED_DOMAINS` is not blank, this option will be ignored. | |||
- `BLOCKED_DOMAINS`: **\<empty\>**: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWED_DOMAINS` is not blank, this option has a higher priority to deny domains. | |||
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291 | |||
- `SKIP_TLS_VERIFY`: **false**: Allow skip tls verify | |||
@@ -335,7 +335,7 @@ IS_INPUT_FILE = false | |||
- `MAX_ATTEMPTS`: **3**: 在迁移过程中的 http/https 请求重试次数。 | |||
- `RETRY_BACKOFF`: **3**: 等待下一次重试的时间,单位秒。 | |||
- `ALLOWED_DOMAINS`: **\<empty\>**: 迁移仓库的域名白名单,默认为空,表示允许从任意域名迁移仓库,多个域名用逗号分隔。 | |||
- `BLOCKED_DOMAINS`: **\<empty\>**: 迁移仓库的域名黑名单,默认为空,多个域名用逗号分隔。如果 `ALLOWED_DOMAINS` 不为空,此选项将会被忽略。 | |||
- `BLOCKED_DOMAINS`: **\<empty\>**: 迁移仓库的域名黑名单,默认为空,多个域名用逗号分隔。如果 `ALLOWED_DOMAINS` 不为空,此选项有更高的优先级拒绝这里的域名。 | |||
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918 | |||
- `SKIP_TLS_VERIFY`: **false**: 允许忽略 TLS 认证 | |||
@@ -14,6 +14,7 @@ import ( | |||
"code.gitea.io/gitea/modules/lfs" | |||
"code.gitea.io/gitea/modules/setting" | |||
api "code.gitea.io/gitea/modules/structs" | |||
"code.gitea.io/gitea/services/migrations" | |||
"github.com/stretchr/testify/assert" | |||
) | |||
@@ -25,6 +26,7 @@ func TestAPIRepoLFSMigrateLocal(t *testing.T) { | |||
oldAllowLocalNetworks := setting.Migrations.AllowLocalNetworks | |||
setting.ImportLocalPaths = true | |||
setting.Migrations.AllowLocalNetworks = true | |||
assert.NoError(t, migrations.Init()) | |||
user := unittest.AssertExistsAndLoadBean(t, &models.User{ID: 1}).(*models.User) | |||
session := loginUser(t, user.Name) | |||
@@ -47,4 +49,5 @@ func TestAPIRepoLFSMigrateLocal(t *testing.T) { | |||
setting.ImportLocalPaths = oldImportLocalPaths | |||
setting.Migrations.AllowLocalNetworks = oldAllowLocalNetworks | |||
assert.NoError(t, migrations.Init()) // reset old migration settings | |||
} |
@@ -331,10 +331,10 @@ func TestAPIRepoMigrate(t *testing.T) { | |||
switch respJSON["message"] { | |||
case "Remote visit addressed rate limitation.": | |||
t.Log("test hit github rate limitation") | |||
case "You are not allowed to import from private IPs.": | |||
case "You can not import from disallowed hosts.": | |||
assert.EqualValues(t, "private-ip", testCase.repoName) | |||
default: | |||
t.Errorf("unexpected error '%v' on url '%s'", respJSON["message"], testCase.cloneURL) | |||
assert.Fail(t, "unexpected error '%v' on url '%s'", respJSON["message"], testCase.cloneURL) | |||
} | |||
} else { | |||
assert.EqualValues(t, testCase.expectedStatus, resp.Code) |
@@ -47,7 +47,7 @@ func TestMirrorPull(t *testing.T) { | |||
ctx := context.Background() | |||
mirror, err := repository.MigrateRepositoryGitData(ctx, user, mirrorRepo, opts) | |||
mirror, err := repository.MigrateRepositoryGitData(ctx, user, mirrorRepo, opts, nil) | |||
assert.NoError(t, err) | |||
gitRepo, err := git.OpenRepository(repoPath) |
@@ -16,6 +16,7 @@ import ( | |||
"code.gitea.io/gitea/modules/git" | |||
"code.gitea.io/gitea/modules/repository" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/services/migrations" | |||
mirror_service "code.gitea.io/gitea/services/mirror" | |||
"github.com/stretchr/testify/assert" | |||
@@ -29,6 +30,7 @@ func testMirrorPush(t *testing.T, u *url.URL) { | |||
defer prepareTestEnv(t)() | |||
setting.Migrations.AllowLocalNetworks = true | |||
assert.NoError(t, migrations.Init()) | |||
user := unittest.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User) | |||
srcRepo := unittest.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository) |
@@ -797,7 +797,6 @@ type ErrInvalidCloneAddr struct { | |||
IsPermissionDenied bool | |||
LocalPath bool | |||
NotResolvedIP bool | |||
PrivateNet string | |||
} | |||
// IsErrInvalidCloneAddr checks if an error is a ErrInvalidCloneAddr. | |||
@@ -810,9 +809,6 @@ func (err *ErrInvalidCloneAddr) Error() string { | |||
if err.NotResolvedIP { | |||
return fmt.Sprintf("migration/cloning from '%s' is not allowed: unknown hostname", err.Host) | |||
} | |||
if len(err.PrivateNet) != 0 { | |||
return fmt.Sprintf("migration/cloning from '%s' is not allowed: the host resolve to a private ip address '%s'", err.Host, err.PrivateNet) | |||
} | |||
if err.IsInvalidPath { | |||
return fmt.Sprintf("migration/cloning from '%s' is not allowed: the provided path is invalid", err.Host) | |||
} |
@@ -13,15 +13,18 @@ import ( | |||
) | |||
// HostMatchList is used to check if a host or IP is in a list. | |||
// If you only need to do wildcard matching, consider to use modules/matchlist | |||
type HostMatchList struct { | |||
hosts []string | |||
SettingKeyHint string | |||
SettingValue string | |||
// builtins networks | |||
builtins []string | |||
// patterns for host names (with wildcard support) | |||
patterns []string | |||
// ipNets is the CIDR network list | |||
ipNets []*net.IPNet | |||
} | |||
// MatchBuiltinAll all hosts are matched | |||
const MatchBuiltinAll = "*" | |||
// MatchBuiltinExternal A valid non-private unicast IP, all hosts on public internet are matched | |||
const MatchBuiltinExternal = "external" | |||
@@ -31,9 +34,13 @@ const MatchBuiltinPrivate = "private" | |||
// MatchBuiltinLoopback 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included. | |||
const MatchBuiltinLoopback = "loopback" | |||
func isBuiltin(s string) bool { | |||
return s == MatchBuiltinExternal || s == MatchBuiltinPrivate || s == MatchBuiltinLoopback | |||
} | |||
// ParseHostMatchList parses the host list HostMatchList | |||
func ParseHostMatchList(hostList string) *HostMatchList { | |||
hl := &HostMatchList{} | |||
func ParseHostMatchList(settingKeyHint string, hostList string) *HostMatchList { | |||
hl := &HostMatchList{SettingKeyHint: settingKeyHint, SettingValue: hostList} | |||
for _, s := range strings.Split(hostList, ",") { | |||
s = strings.ToLower(strings.TrimSpace(s)) | |||
if s == "" { | |||
@@ -42,53 +49,106 @@ func ParseHostMatchList(hostList string) *HostMatchList { | |||
_, ipNet, err := net.ParseCIDR(s) | |||
if err == nil { | |||
hl.ipNets = append(hl.ipNets, ipNet) | |||
} else if isBuiltin(s) { | |||
hl.builtins = append(hl.builtins, s) | |||
} else { | |||
hl.hosts = append(hl.hosts, s) | |||
hl.patterns = append(hl.patterns, s) | |||
} | |||
} | |||
return hl | |||
} | |||
// MatchesHostOrIP checks if the host or IP matches an allow/deny(block) list | |||
func (hl *HostMatchList) MatchesHostOrIP(host string, ip net.IP) bool { | |||
var matched bool | |||
host = strings.ToLower(host) | |||
ipStr := ip.String() | |||
loop: | |||
for _, hostInList := range hl.hosts { | |||
switch hostInList { | |||
case "": | |||
// ParseSimpleMatchList parse a simple matchlist (no built-in networks, no CIDR support, only wildcard pattern match) | |||
func ParseSimpleMatchList(settingKeyHint string, matchList string) *HostMatchList { | |||
hl := &HostMatchList{ | |||
SettingKeyHint: settingKeyHint, | |||
SettingValue: matchList, | |||
} | |||
for _, s := range strings.Split(matchList, ",") { | |||
s = strings.ToLower(strings.TrimSpace(s)) | |||
if s == "" { | |||
continue | |||
case MatchBuiltinAll: | |||
matched = true | |||
break loop | |||
} | |||
// we keep the same result as old `matchlist`, so no builtin/CIDR support here, we only match wildcard patterns | |||
hl.patterns = append(hl.patterns, s) | |||
} | |||
return hl | |||
} | |||
// AppendBuiltin appends more builtins to match | |||
func (hl *HostMatchList) AppendBuiltin(builtin string) { | |||
hl.builtins = append(hl.builtins, builtin) | |||
} | |||
// IsEmpty checks if the checklist is empty | |||
func (hl *HostMatchList) IsEmpty() bool { | |||
return hl == nil || (len(hl.builtins) == 0 && len(hl.patterns) == 0 && len(hl.ipNets) == 0) | |||
} | |||
func (hl *HostMatchList) checkPattern(host string) bool { | |||
host = strings.ToLower(strings.TrimSpace(host)) | |||
for _, pattern := range hl.patterns { | |||
if matched, _ := filepath.Match(pattern, host); matched { | |||
return true | |||
} | |||
} | |||
return false | |||
} | |||
func (hl *HostMatchList) checkIP(ip net.IP) bool { | |||
for _, pattern := range hl.patterns { | |||
if pattern == "*" { | |||
return true | |||
} | |||
} | |||
for _, builtin := range hl.builtins { | |||
switch builtin { | |||
case MatchBuiltinExternal: | |||
if matched = ip.IsGlobalUnicast() && !util.IsIPPrivate(ip); matched { | |||
break loop | |||
if ip.IsGlobalUnicast() && !util.IsIPPrivate(ip) { | |||
return true | |||
} | |||
case MatchBuiltinPrivate: | |||
if matched = util.IsIPPrivate(ip); matched { | |||
break loop | |||
if util.IsIPPrivate(ip) { | |||
return true | |||
} | |||
case MatchBuiltinLoopback: | |||
if matched = ip.IsLoopback(); matched { | |||
break loop | |||
} | |||
default: | |||
if matched, _ = filepath.Match(hostInList, host); matched { | |||
break loop | |||
} | |||
if matched, _ = filepath.Match(hostInList, ipStr); matched { | |||
break loop | |||
if ip.IsLoopback() { | |||
return true | |||
} | |||
} | |||
} | |||
if !matched { | |||
for _, ipNet := range hl.ipNets { | |||
if matched = ipNet.Contains(ip); matched { | |||
break | |||
} | |||
for _, ipNet := range hl.ipNets { | |||
if ipNet.Contains(ip) { | |||
return true | |||
} | |||
} | |||
return matched | |||
return false | |||
} | |||
// MatchHostName checks if the host matches an allow/deny(block) list | |||
func (hl *HostMatchList) MatchHostName(host string) bool { | |||
if hl == nil { | |||
return false | |||
} | |||
if hl.checkPattern(host) { | |||
return true | |||
} | |||
if ip := net.ParseIP(host); ip != nil { | |||
return hl.checkIP(ip) | |||
} | |||
return false | |||
} | |||
// MatchIPAddr checks if the IP matches an allow/deny(block) list, it's safe to pass `nil` to `ip` | |||
func (hl *HostMatchList) MatchIPAddr(ip net.IP) bool { | |||
if hl == nil { | |||
return false | |||
} | |||
host := ip.String() // nil-safe, we will get "<nil>" if ip is nil | |||
return hl.checkPattern(host) || hl.checkIP(ip) | |||
} | |||
// MatchHostOrIP checks if the host or IP matches an allow/deny(block) list | |||
func (hl *HostMatchList) MatchHostOrIP(host string, ip net.IP) bool { | |||
return hl.MatchHostName(host) || hl.MatchIPAddr(ip) | |||
} |
@@ -20,17 +20,28 @@ func TestHostOrIPMatchesList(t *testing.T) { | |||
// for IPv6: "::1" is loopback, "fd00::/8" is private | |||
hl := ParseHostMatchList("private, External, *.myDomain.com, 169.254.1.0/24") | |||
hl := ParseHostMatchList("", "private, External, *.myDomain.com, 169.254.1.0/24") | |||
test := func(cases []tc) { | |||
for _, c := range cases { | |||
assert.Equalf(t, c.expected, hl.MatchHostOrIP(c.host, c.ip), "case domain=%s, ip=%v, expected=%v", c.host, c.ip, c.expected) | |||
} | |||
} | |||
cases := []tc{ | |||
{"", net.IPv4zero, false}, | |||
{"", net.IPv6zero, false}, | |||
{"", net.ParseIP("127.0.0.1"), false}, | |||
{"127.0.0.1", nil, false}, | |||
{"", net.ParseIP("::1"), false}, | |||
{"", net.ParseIP("10.0.1.1"), true}, | |||
{"10.0.1.1", nil, true}, | |||
{"", net.ParseIP("192.168.1.1"), true}, | |||
{"192.168.1.1", nil, true}, | |||
{"", net.ParseIP("fd00::1"), true}, | |||
{"fd00::1", nil, true}, | |||
{"", net.ParseIP("8.8.8.8"), true}, | |||
{"", net.ParseIP("1001::1"), true}, | |||
@@ -39,13 +50,13 @@ func TestHostOrIPMatchesList(t *testing.T) { | |||
{"sub.mydomain.com", net.IPv4zero, true}, | |||
{"", net.ParseIP("169.254.1.1"), true}, | |||
{"169.254.1.1", nil, true}, | |||
{"", net.ParseIP("169.254.2.2"), false}, | |||
{"169.254.2.2", nil, false}, | |||
} | |||
for _, c := range cases { | |||
assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) | |||
} | |||
test(cases) | |||
hl = ParseHostMatchList("loopback") | |||
hl = ParseHostMatchList("", "loopback") | |||
cases = []tc{ | |||
{"", net.IPv4zero, false}, | |||
{"", net.ParseIP("127.0.0.1"), true}, | |||
@@ -59,11 +70,9 @@ func TestHostOrIPMatchesList(t *testing.T) { | |||
{"mydomain.com", net.IPv4zero, false}, | |||
} | |||
for _, c := range cases { | |||
assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) | |||
} | |||
test(cases) | |||
hl = ParseHostMatchList("private") | |||
hl = ParseHostMatchList("", "private") | |||
cases = []tc{ | |||
{"", net.IPv4zero, false}, | |||
{"", net.ParseIP("127.0.0.1"), false}, | |||
@@ -77,11 +86,9 @@ func TestHostOrIPMatchesList(t *testing.T) { | |||
{"mydomain.com", net.IPv4zero, false}, | |||
} | |||
for _, c := range cases { | |||
assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) | |||
} | |||
test(cases) | |||
hl = ParseHostMatchList("external") | |||
hl = ParseHostMatchList("", "external") | |||
cases = []tc{ | |||
{"", net.IPv4zero, false}, | |||
{"", net.ParseIP("127.0.0.1"), false}, | |||
@@ -95,11 +102,9 @@ func TestHostOrIPMatchesList(t *testing.T) { | |||
{"mydomain.com", net.IPv4zero, false}, | |||
} | |||
for _, c := range cases { | |||
assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) | |||
} | |||
test(cases) | |||
hl = ParseHostMatchList("*") | |||
hl = ParseHostMatchList("", "*") | |||
cases = []tc{ | |||
{"", net.IPv4zero, true}, | |||
{"", net.ParseIP("127.0.0.1"), true}, | |||
@@ -113,7 +118,43 @@ func TestHostOrIPMatchesList(t *testing.T) { | |||
{"mydomain.com", net.IPv4zero, true}, | |||
} | |||
for _, c := range cases { | |||
assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip) | |||
test(cases) | |||
// built-in network names can be escaped (warping the first char with `[]`) to be used as a real host name | |||
// this mechanism is reversed for internal usage only (maybe for some rare cases), it's not supposed to be used by end users | |||
// a real user should never use loopback/private/external as their host names | |||
hl = ParseHostMatchList("", "loopback, [p]rivate") | |||
cases = []tc{ | |||
{"loopback", nil, false}, | |||
{"", net.ParseIP("127.0.0.1"), true}, | |||
{"private", nil, true}, | |||
{"", net.ParseIP("192.168.1.1"), false}, | |||
} | |||
test(cases) | |||
hl = ParseSimpleMatchList("", "loopback, *.domain.com") | |||
cases = []tc{ | |||
{"loopback", nil, true}, | |||
{"", net.ParseIP("127.0.0.1"), false}, | |||
{"sub.domain.com", nil, true}, | |||
{"other.com", nil, false}, | |||
{"", net.ParseIP("1.1.1.1"), false}, | |||
} | |||
test(cases) | |||
hl = ParseSimpleMatchList("", "external") | |||
cases = []tc{ | |||
{"", net.ParseIP("192.168.1.1"), false}, | |||
{"", net.ParseIP("1.1.1.1"), false}, | |||
{"external", nil, true}, | |||
} | |||
test(cases) | |||
hl = ParseSimpleMatchList("", "") | |||
cases = []tc{ | |||
{"", net.ParseIP("192.168.1.1"), false}, | |||
{"", net.ParseIP("1.1.1.1"), false}, | |||
{"external", nil, false}, | |||
} | |||
test(cases) | |||
} |
@@ -0,0 +1,58 @@ | |||
// Copyright 2021 The Gitea Authors. All rights reserved. | |||
// Use of this source code is governed by a MIT-style | |||
// license that can be found in the LICENSE file. | |||
package hostmatcher | |||
import ( | |||
"context" | |||
"fmt" | |||
"net" | |||
"syscall" | |||
"time" | |||
) | |||
// NewDialContext returns a DialContext for Transport, the DialContext will do allow/block list check | |||
func NewDialContext(usage string, allowList *HostMatchList, blockList *HostMatchList) func(ctx context.Context, network, addr string) (net.Conn, error) { | |||
// How Go HTTP Client works with redirection: | |||
// transport.RoundTrip URL=http://domain.com, Host=domain.com | |||
// transport.DialContext addrOrHost=domain.com:80 | |||
// dialer.Control tcp4:11.22.33.44:80 | |||
// transport.RoundTrip URL=http://www.domain.com/, Host=(empty here, in the direction, HTTP client doesn't fill the Host field) | |||
// transport.DialContext addrOrHost=domain.com:80 | |||
// dialer.Control tcp4:11.22.33.44:80 | |||
return func(ctx context.Context, network, addrOrHost string) (net.Conn, error) { | |||
dialer := net.Dialer{ | |||
// default values comes from http.DefaultTransport | |||
Timeout: 30 * time.Second, | |||
KeepAlive: 30 * time.Second, | |||
Control: func(network, ipAddr string, c syscall.RawConn) (err error) { | |||
var host string | |||
if host, _, err = net.SplitHostPort(addrOrHost); err != nil { | |||
return err | |||
} | |||
// in Control func, the addr was already resolved to IP:PORT format, there is no cost to do ResolveTCPAddr here | |||
tcpAddr, err := net.ResolveTCPAddr(network, ipAddr) | |||
if err != nil { | |||
return fmt.Errorf("%s can only call HTTP servers via TCP, deny '%s(%s:%s)', err=%v", usage, host, network, ipAddr, err) | |||
} | |||
var blockedError error | |||
if blockList.MatchHostOrIP(host, tcpAddr.IP) { | |||
blockedError = fmt.Errorf("%s can not call blocked HTTP servers (check your %s setting), deny '%s(%s)'", usage, blockList.SettingKeyHint, host, ipAddr) | |||
} | |||
// if we have an allow-list, check the allow-list first | |||
if !allowList.IsEmpty() { | |||
if !allowList.MatchHostOrIP(host, tcpAddr.IP) { | |||
return fmt.Errorf("%s can only call allowed HTTP servers (check your %s setting), deny '%s(%s)'", usage, allowList.SettingKeyHint, host, ipAddr) | |||
} | |||
} | |||
// otherwise, we always follow the blocked list | |||
return blockedError | |||
}, | |||
} | |||
return dialer.DialContext(ctx, network, addrOrHost) | |||
} | |||
} |
@@ -7,6 +7,7 @@ package lfs | |||
import ( | |||
"context" | |||
"io" | |||
"net/http" | |||
"net/url" | |||
) | |||
@@ -24,9 +25,9 @@ type Client interface { | |||
} | |||
// NewClient creates a LFS client | |||
func NewClient(endpoint *url.URL, skipTLSVerify bool) Client { | |||
func NewClient(endpoint *url.URL, httpTransport *http.Transport) Client { | |||
if endpoint.Scheme == "file" { | |||
return newFilesystemClient(endpoint) | |||
} | |||
return newHTTPClient(endpoint, skipTLSVerify) | |||
return newHTTPClient(endpoint, httpTransport) | |||
} |
@@ -13,10 +13,10 @@ import ( | |||
func TestNewClient(t *testing.T) { | |||
u, _ := url.Parse("file:///test") | |||
c := NewClient(u, true) | |||
c := NewClient(u, nil) | |||
assert.IsType(t, &FilesystemClient{}, c) | |||
u, _ = url.Parse("https://test.com/lfs") | |||
c = NewClient(u, true) | |||
c = NewClient(u, nil) | |||
assert.IsType(t, &HTTPClient{}, c) | |||
} |
@@ -7,7 +7,6 @@ package lfs | |||
import ( | |||
"bytes" | |||
"context" | |||
"crypto/tls" | |||
"errors" | |||
"fmt" | |||
"net/http" | |||
@@ -34,12 +33,15 @@ func (c *HTTPClient) BatchSize() int { | |||
return batchSize | |||
} | |||
func newHTTPClient(endpoint *url.URL, skipTLSVerify bool) *HTTPClient { | |||
func newHTTPClient(endpoint *url.URL, httpTransport *http.Transport) *HTTPClient { | |||
if httpTransport == nil { | |||
httpTransport = &http.Transport{ | |||
Proxy: proxy.Proxy(), | |||
} | |||
} | |||
hc := &http.Client{ | |||
Transport: &http.Transport{ | |||
TLSClientConfig: &tls.Config{InsecureSkipVerify: skipTLSVerify}, | |||
Proxy: proxy.Proxy(), | |||
}, | |||
Transport: httpTransport, | |||
} | |||
client := &HTTPClient{ |
@@ -1,46 +0,0 @@ | |||
// Copyright 2019 The Gitea Authors. All rights reserved. | |||
// Use of this source code is governed by a MIT-style | |||
// license that can be found in the LICENSE file. | |||
package matchlist | |||
import ( | |||
"strings" | |||
"github.com/gobwas/glob" | |||
) | |||
// Matchlist represents a block or allow list | |||
type Matchlist struct { | |||
ruleGlobs []glob.Glob | |||
} | |||
// NewMatchlist creates a new block or allow list | |||
func NewMatchlist(rules ...string) (*Matchlist, error) { | |||
for i := range rules { | |||
rules[i] = strings.ToLower(rules[i]) | |||
} | |||
list := Matchlist{ | |||
ruleGlobs: make([]glob.Glob, 0, len(rules)), | |||
} | |||
for _, rule := range rules { | |||
rg, err := glob.Compile(rule) | |||
if err != nil { | |||
return nil, err | |||
} | |||
list.ruleGlobs = append(list.ruleGlobs, rg) | |||
} | |||
return &list, nil | |||
} | |||
// Match will matches | |||
func (b *Matchlist) Match(u string) bool { | |||
for _, r := range b.ruleGlobs { | |||
if r.Match(u) { | |||
return true | |||
} | |||
} | |||
return false | |||
} |
@@ -8,7 +8,7 @@ import ( | |||
"context" | |||
"fmt" | |||
"io" | |||
"net/url" | |||
"net/http" | |||
"path" | |||
"strings" | |||
"time" | |||
@@ -46,7 +46,10 @@ func WikiRemoteURL(remote string) string { | |||
} | |||
// MigrateRepositoryGitData starts migrating git related data after created migrating repository | |||
func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models.Repository, opts migration.MigrateOptions) (*models.Repository, error) { | |||
func MigrateRepositoryGitData(ctx context.Context, u *models.User, | |||
repo *models.Repository, opts migration.MigrateOptions, | |||
httpTransport *http.Transport, | |||
) (*models.Repository, error) { | |||
repoPath := models.RepoPath(u.Name, opts.RepoName) | |||
if u.IsOrganization() { | |||
@@ -141,8 +144,9 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models. | |||
} | |||
if opts.LFS { | |||
ep := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint) | |||
if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, ep, setting.Migrations.SkipTLSVerify); err != nil { | |||
endpoint := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint) | |||
lfsClient := lfs.NewClient(endpoint, httpTransport) | |||
if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, lfsClient); err != nil { | |||
log.Error("Failed to store missing LFS objects for repository: %v", err) | |||
} | |||
} | |||
@@ -336,8 +340,7 @@ func PushUpdateAddTag(repo *models.Repository, gitRepo *git.Repository, tagName | |||
} | |||
// StoreMissingLfsObjectsInRepository downloads missing LFS objects | |||
func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, endpoint *url.URL, skipTLSVerify bool) error { | |||
client := lfs.NewClient(endpoint, skipTLSVerify) | |||
func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, lfsClient lfs.Client) error { | |||
contentStore := lfs.NewContentStore() | |||
pointerChan := make(chan lfs.PointerBlob) | |||
@@ -345,7 +348,7 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi | |||
go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan) | |||
downloadObjects := func(pointers []lfs.Pointer) error { | |||
err := client.Download(ctx, pointers, func(p lfs.Pointer, content io.ReadCloser, objectError error) error { | |||
err := lfsClient.Download(ctx, pointers, func(p lfs.Pointer, content io.ReadCloser, objectError error) error { | |||
if objectError != nil { | |||
return objectError | |||
} | |||
@@ -411,7 +414,7 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi | |||
} | |||
batch = append(batch, pointerBlob.Pointer) | |||
if len(batch) >= client.BatchSize() { | |||
if len(batch) >= lfsClient.BatchSize() { | |||
if err := downloadObjects(batch); err != nil { | |||
return err | |||
} |
@@ -4,17 +4,13 @@ | |||
package setting | |||
import ( | |||
"strings" | |||
) | |||
var ( | |||
// Migrations settings | |||
Migrations = struct { | |||
MaxAttempts int | |||
RetryBackoff int | |||
AllowedDomains []string | |||
BlockedDomains []string | |||
AllowedDomains string | |||
BlockedDomains string | |||
AllowLocalNetworks bool | |||
SkipTLSVerify bool | |||
}{ | |||
@@ -28,15 +24,8 @@ func newMigrationsService() { | |||
Migrations.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Migrations.MaxAttempts) | |||
Migrations.RetryBackoff = sec.Key("RETRY_BACKOFF").MustInt(Migrations.RetryBackoff) | |||
Migrations.AllowedDomains = sec.Key("ALLOWED_DOMAINS").Strings(",") | |||
for i := range Migrations.AllowedDomains { | |||
Migrations.AllowedDomains[i] = strings.ToLower(Migrations.AllowedDomains[i]) | |||
} | |||
Migrations.BlockedDomains = sec.Key("BLOCKED_DOMAINS").Strings(",") | |||
for i := range Migrations.BlockedDomains { | |||
Migrations.BlockedDomains[i] = strings.ToLower(Migrations.BlockedDomains[i]) | |||
} | |||
Migrations.AllowedDomains = sec.Key("ALLOWED_DOMAINS").MustString("") | |||
Migrations.BlockedDomains = sec.Key("BLOCKED_DOMAINS").MustString("") | |||
Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false) | |||
Migrations.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool(false) | |||
} |
@@ -7,7 +7,6 @@ package setting | |||
import ( | |||
"net/url" | |||
"code.gitea.io/gitea/modules/hostmatcher" | |||
"code.gitea.io/gitea/modules/log" | |||
) | |||
@@ -17,7 +16,7 @@ var ( | |||
QueueLength int | |||
DeliverTimeout int | |||
SkipTLSVerify bool | |||
AllowedHostList *hostmatcher.HostMatchList | |||
AllowedHostList string | |||
Types []string | |||
PagingNum int | |||
ProxyURL string | |||
@@ -38,7 +37,7 @@ func newWebhookService() { | |||
Webhook.QueueLength = sec.Key("QUEUE_LENGTH").MustInt(1000) | |||
Webhook.DeliverTimeout = sec.Key("DELIVER_TIMEOUT").MustInt(5) | |||
Webhook.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool() | |||
Webhook.AllowedHostList = hostmatcher.ParseHostMatchList(sec.Key("ALLOWED_HOST_LIST").MustString(hostmatcher.MatchBuiltinExternal)) | |||
Webhook.AllowedHostList = sec.Key("ALLOWED_HOST_LIST").MustString("") | |||
Webhook.Types = []string{"gitea", "gogs", "slack", "discord", "dingtalk", "telegram", "msteams", "feishu", "matrix", "wechatwork"} | |||
Webhook.PagingNum = sec.Key("PAGING_NUM").MustInt(10) | |||
Webhook.ProxyURL = sec.Key("PROXY_URL").MustString("") |
@@ -899,8 +899,7 @@ migrate.clone_address_desc = The HTTP(S) or Git 'clone' URL of an existing repos | |||
migrate.github_token_desc = You can put one or more tokens with comma separated here to make migrating faster because of Github API rate limit. WARN: Abusing this feature may violate the service provider's policy and lead to account blocking. | |||
migrate.clone_local_path = or a local server path | |||
migrate.permission_denied = You are not allowed to import local repositories. | |||
migrate.permission_denied_blocked = You are not allowed to import from blocked hosts. | |||
migrate.permission_denied_private_ip = You are not allowed to import from private IPs. | |||
migrate.permission_denied_blocked = You can not import from disallowed hosts, please ask the admin to check ALLOWED_DOMAINS/ALLOW_LOCALNETWORKS/BLOCKED_DOMAINS settings. | |||
migrate.invalid_local_path = "The local path is invalid. It does not exist or is not a directory." | |||
migrate.invalid_lfs_endpoint = The LFS endpoint is not valid. | |||
migrate.failed = Migration failed: %v |
@@ -253,10 +253,8 @@ func handleRemoteAddrError(ctx *context.APIContext, err error) { | |||
case addrErr.IsPermissionDenied: | |||
if addrErr.LocalPath { | |||
ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import local repositories.") | |||
} else if len(addrErr.PrivateNet) == 0 { | |||
ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import from blocked hosts.") | |||
} else { | |||
ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import from private IPs.") | |||
ctx.Error(http.StatusUnprocessableEntity, "", "You can not import from disallowed hosts.") | |||
} | |||
case addrErr.IsInvalidPath: | |||
ctx.Error(http.StatusUnprocessableEntity, "", "Invalid local path, it does not exist or not a directory.") |
@@ -128,10 +128,8 @@ func handleMigrateRemoteAddrError(ctx *context.Context, err error, tpl base.TplN | |||
case addrErr.IsPermissionDenied: | |||
if addrErr.LocalPath { | |||
ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied"), tpl, form) | |||
} else if len(addrErr.PrivateNet) == 0 { | |||
ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tpl, form) | |||
} else { | |||
ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_private_ip"), tpl, form) | |||
ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tpl, form) | |||
} | |||
case addrErr.IsInvalidPath: | |||
ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_local_path"), tpl, form) |
@@ -750,10 +750,8 @@ func handleSettingRemoteAddrError(ctx *context.Context, err error, form *forms.R | |||
case addrErr.IsPermissionDenied: | |||
if addrErr.LocalPath { | |||
ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied"), tplSettingsOptions, form) | |||
} else if len(addrErr.PrivateNet) == 0 { | |||
ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tplSettingsOptions, form) | |||
} else { | |||
ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_private_ip"), tplSettingsOptions, form) | |||
ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tplSettingsOptions, form) | |||
} | |||
case addrErr.IsInvalidPath: | |||
ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_local_path"), tplSettingsOptions, form) |
@@ -6,7 +6,6 @@ package migrations | |||
import ( | |||
"context" | |||
"crypto/tls" | |||
"errors" | |||
"fmt" | |||
"io" | |||
@@ -18,8 +17,6 @@ import ( | |||
admin_model "code.gitea.io/gitea/models/admin" | |||
"code.gitea.io/gitea/modules/log" | |||
base "code.gitea.io/gitea/modules/migration" | |||
"code.gitea.io/gitea/modules/proxy" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/modules/structs" | |||
gitea_sdk "code.gitea.io/sdk/gitea" | |||
@@ -90,12 +87,7 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo | |||
gitea_sdk.SetToken(token), | |||
gitea_sdk.SetBasicAuth(username, password), | |||
gitea_sdk.SetContext(ctx), | |||
gitea_sdk.SetHTTPClient(&http.Client{ | |||
Transport: &http.Transport{ | |||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}, | |||
Proxy: proxy.Proxy(), | |||
}, | |||
}), | |||
gitea_sdk.SetHTTPClient(NewMigrationHTTPClient()), | |||
) | |||
if err != nil { | |||
log.Error(fmt.Sprintf("Failed to create NewGiteaDownloader for: %s. Error: %v", baseURL, err)) | |||
@@ -275,12 +267,7 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele | |||
Created: rel.CreatedAt, | |||
} | |||
httpClient := &http.Client{ | |||
Transport: &http.Transport{ | |||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}, | |||
Proxy: proxy.Proxy(), | |||
}, | |||
} | |||
httpClient := NewMigrationHTTPClient() | |||
for _, asset := range rel.Attachments { | |||
size := int(asset.Size) |
@@ -125,7 +125,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate | |||
Wiki: opts.Wiki, | |||
Releases: opts.Releases, // if didn't get releases, then sync them from tags | |||
MirrorInterval: opts.MirrorInterval, | |||
}) | |||
}, NewMigrationHTTPTransport()) | |||
g.repo = r | |||
if err != nil { |
@@ -7,7 +7,6 @@ package migrations | |||
import ( | |||
"context" | |||
"crypto/tls" | |||
"fmt" | |||
"io" | |||
"net/http" | |||
@@ -19,7 +18,6 @@ import ( | |||
"code.gitea.io/gitea/modules/log" | |||
base "code.gitea.io/gitea/modules/migration" | |||
"code.gitea.io/gitea/modules/proxy" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/modules/structs" | |||
"code.gitea.io/gitea/modules/util" | |||
@@ -100,12 +98,7 @@ func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, tok | |||
) | |||
var client = &http.Client{ | |||
Transport: &oauth2.Transport{ | |||
Base: &http.Transport{ | |||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}, | |||
Proxy: func(req *http.Request) (*url.URL, error) { | |||
return proxy.Proxy()(req) | |||
}, | |||
}, | |||
Base: NewMigrationHTTPTransport(), | |||
Source: oauth2.ReuseTokenSource(nil, ts), | |||
}, | |||
} | |||
@@ -113,14 +106,13 @@ func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, tok | |||
downloader.addClient(client, baseURL) | |||
} | |||
} else { | |||
var transport = NewMigrationHTTPTransport() | |||
transport.Proxy = func(req *http.Request) (*url.URL, error) { | |||
req.SetBasicAuth(userName, password) | |||
return proxy.Proxy()(req) | |||
} | |||
var client = &http.Client{ | |||
Transport: &http.Transport{ | |||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}, | |||
Proxy: func(req *http.Request) (*url.URL, error) { | |||
req.SetBasicAuth(userName, password) | |||
return proxy.Proxy()(req) | |||
}, | |||
}, | |||
Transport: transport, | |||
} | |||
downloader.addClient(client, baseURL) | |||
} | |||
@@ -316,12 +308,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) | |||
r.Published = rel.PublishedAt.Time | |||
} | |||
httpClient := &http.Client{ | |||
Transport: &http.Transport{ | |||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}, | |||
Proxy: proxy.Proxy(), | |||
}, | |||
} | |||
httpClient := NewMigrationHTTPClient() | |||
for _, asset := range rel.Assets { | |||
var assetID = *asset.ID // Don't optimize this, for closure we need a local variable |
@@ -6,7 +6,6 @@ package migrations | |||
import ( | |||
"context" | |||
"crypto/tls" | |||
"errors" | |||
"fmt" | |||
"io" | |||
@@ -18,8 +17,6 @@ import ( | |||
"code.gitea.io/gitea/modules/log" | |||
base "code.gitea.io/gitea/modules/migration" | |||
"code.gitea.io/gitea/modules/proxy" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/modules/structs" | |||
"github.com/xanzy/go-gitlab" | |||
@@ -77,16 +74,11 @@ type GitlabDownloader struct { | |||
// Use either a username/password, personal token entered into the username field, or anonymous/public access | |||
// Note: Public access only allows very basic access | |||
func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GitlabDownloader, error) { | |||
gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(&http.Client{ | |||
Transport: &http.Transport{ | |||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}, | |||
Proxy: proxy.Proxy(), | |||
}, | |||
})) | |||
gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(NewMigrationHTTPClient())) | |||
// Only use basic auth if token is blank and password is NOT | |||
// Basic auth will fail with empty strings, but empty token will allow anonymous public API usage | |||
if token == "" && password != "" { | |||
gitlabClient, err = gitlab.NewBasicAuthClient(username, password, gitlab.WithBaseURL(baseURL)) | |||
gitlabClient, err = gitlab.NewBasicAuthClient(username, password, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(NewMigrationHTTPClient())) | |||
} | |||
if err != nil { | |||
@@ -300,12 +292,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea | |||
PublisherName: rel.Author.Username, | |||
} | |||
httpClient := &http.Client{ | |||
Transport: &http.Transport{ | |||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}, | |||
Proxy: proxy.Proxy(), | |||
}, | |||
} | |||
httpClient := NewMigrationHTTPClient() | |||
for k, asset := range rel.Assets.Links { | |||
r.Assets = append(r.Assets, &base.ReleaseAsset{ |
@@ -6,7 +6,6 @@ package migrations | |||
import ( | |||
"context" | |||
"crypto/tls" | |||
"fmt" | |||
"net/http" | |||
"net/url" | |||
@@ -16,7 +15,6 @@ import ( | |||
"code.gitea.io/gitea/modules/log" | |||
base "code.gitea.io/gitea/modules/migration" | |||
"code.gitea.io/gitea/modules/proxy" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/modules/structs" | |||
"github.com/gogs/go-gogs-client" | |||
@@ -97,13 +95,12 @@ func NewGogsDownloader(ctx context.Context, baseURL, userName, password, token, | |||
client = gogs.NewClient(baseURL, token) | |||
downloader.userName = token | |||
} else { | |||
downloader.transport = &http.Transport{ | |||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}, | |||
Proxy: func(req *http.Request) (*url.URL, error) { | |||
req.SetBasicAuth(userName, password) | |||
return proxy.Proxy()(req) | |||
}, | |||
var transport = NewMigrationHTTPTransport() | |||
transport.Proxy = func(req *http.Request) (*url.URL, error) { | |||
req.SetBasicAuth(userName, password) | |||
return proxy.Proxy()(req) | |||
} | |||
downloader.transport = transport | |||
client = gogs.NewClient(baseURL, "") | |||
client.SetHTTPClient(&http.Client{ |
@@ -0,0 +1,30 @@ | |||
// Copyright 2021 The Gitea Authors. All rights reserved. | |||
// Use of this source code is governed by a MIT-style | |||
// license that can be found in the LICENSE file. | |||
package migrations | |||
import ( | |||
"crypto/tls" | |||
"net/http" | |||
"code.gitea.io/gitea/modules/hostmatcher" | |||
"code.gitea.io/gitea/modules/proxy" | |||
"code.gitea.io/gitea/modules/setting" | |||
) | |||
// NewMigrationHTTPClient returns a HTTP client for migration | |||
func NewMigrationHTTPClient() *http.Client { | |||
return &http.Client{ | |||
Transport: NewMigrationHTTPTransport(), | |||
} | |||
} | |||
// NewMigrationHTTPTransport returns a HTTP transport for migration | |||
func NewMigrationHTTPTransport() *http.Transport { | |||
return &http.Transport{ | |||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}, | |||
Proxy: proxy.Proxy(), | |||
DialContext: hostmatcher.NewDialContext("migration", allowList, blockList), | |||
} | |||
} |
@@ -15,8 +15,8 @@ import ( | |||
"code.gitea.io/gitea/models" | |||
admin_model "code.gitea.io/gitea/models/admin" | |||
"code.gitea.io/gitea/modules/hostmatcher" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/matchlist" | |||
base "code.gitea.io/gitea/modules/migration" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/modules/util" | |||
@@ -28,8 +28,8 @@ type MigrateOptions = base.MigrateOptions | |||
var ( | |||
factories []base.DownloaderFactory | |||
allowList *matchlist.Matchlist | |||
blockList *matchlist.Matchlist | |||
allowList *hostmatcher.HostMatchList | |||
blockList *hostmatcher.HostMatchList | |||
) | |||
// RegisterDownloaderFactory registers a downloader factory | |||
@@ -73,30 +73,35 @@ func IsMigrateURLAllowed(remoteURL string, doer *models.User) error { | |||
return &models.ErrInvalidCloneAddr{Host: u.Host, IsProtocolInvalid: true, IsPermissionDenied: true, IsURLError: true} | |||
} | |||
host := strings.ToLower(u.Host) | |||
if len(setting.Migrations.AllowedDomains) > 0 { | |||
if !allowList.Match(host) { | |||
return &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true} | |||
} | |||
} else { | |||
if blockList.Match(host) { | |||
return &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true} | |||
} | |||
hostName, _, err := net.SplitHostPort(u.Host) | |||
if err != nil { | |||
// u.Host can be "host" or "host:port" | |||
err = nil //nolint | |||
hostName = u.Host | |||
} | |||
addrList, err := net.LookupIP(hostName) | |||
if err != nil { | |||
return &models.ErrInvalidCloneAddr{Host: u.Host, NotResolvedIP: true} | |||
} | |||
if !setting.Migrations.AllowLocalNetworks { | |||
addrList, err := net.LookupIP(strings.Split(u.Host, ":")[0]) | |||
if err != nil { | |||
return &models.ErrInvalidCloneAddr{Host: u.Host, NotResolvedIP: true} | |||
} | |||
for _, addr := range addrList { | |||
if util.IsIPPrivate(addr) || !addr.IsGlobalUnicast() { | |||
return &models.ErrInvalidCloneAddr{Host: u.Host, PrivateNet: addr.String(), IsPermissionDenied: true} | |||
} | |||
var ipAllowed bool | |||
var ipBlocked bool | |||
for _, addr := range addrList { | |||
ipAllowed = ipAllowed || allowList.MatchIPAddr(addr) | |||
ipBlocked = ipBlocked || blockList.MatchIPAddr(addr) | |||
} | |||
var blockedError error | |||
if blockList.MatchHostName(hostName) || ipBlocked { | |||
blockedError = &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true} | |||
} | |||
// if we have an allow-list, check the allow-list first | |||
if !allowList.IsEmpty() { | |||
if !allowList.MatchHostName(hostName) && !ipAllowed { | |||
return &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true} | |||
} | |||
} | |||
return nil | |||
// otherwise, we always follow the blocked list | |||
return blockedError | |||
} | |||
// MigrateRepository migrate repository according MigrateOptions | |||
@@ -462,16 +467,18 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts | |||
// Init migrations service | |||
func Init() error { | |||
var err error | |||
allowList, err = matchlist.NewMatchlist(setting.Migrations.AllowedDomains...) | |||
if err != nil { | |||
return fmt.Errorf("init migration allowList domains failed: %v", err) | |||
} | |||
// TODO: maybe we can deprecate these legacy ALLOWED_DOMAINS/ALLOW_LOCALNETWORKS/BLOCKED_DOMAINS, use ALLOWED_HOST_LIST/BLOCKED_HOST_LIST instead | |||
blockList, err = matchlist.NewMatchlist(setting.Migrations.BlockedDomains...) | |||
if err != nil { | |||
return fmt.Errorf("init migration blockList domains failed: %v", err) | |||
} | |||
blockList = hostmatcher.ParseSimpleMatchList("migrations.BLOCKED_DOMAINS", setting.Migrations.BlockedDomains) | |||
allowList = hostmatcher.ParseSimpleMatchList("migrations.ALLOWED_DOMAINS/ALLOW_LOCALNETWORKS", setting.Migrations.AllowedDomains) | |||
if allowList.IsEmpty() { | |||
// the default policy is that migration module can access external hosts | |||
allowList.AppendBuiltin(hostmatcher.MatchBuiltinExternal) | |||
} | |||
if setting.Migrations.AllowLocalNetworks { | |||
allowList.AppendBuiltin(hostmatcher.MatchBuiltinPrivate) | |||
allowList.AppendBuiltin(hostmatcher.MatchBuiltinLoopback) | |||
} | |||
return nil | |||
} |
@@ -21,7 +21,8 @@ func TestMigrateWhiteBlocklist(t *testing.T) { | |||
adminUser := unittest.AssertExistsAndLoadBean(t, &models.User{Name: "user1"}).(*models.User) | |||
nonAdminUser := unittest.AssertExistsAndLoadBean(t, &models.User{Name: "user2"}).(*models.User) | |||
setting.Migrations.AllowedDomains = []string{"github.com"} | |||
setting.Migrations.AllowedDomains = "github.com" | |||
setting.Migrations.AllowLocalNetworks = false | |||
assert.NoError(t, Init()) | |||
err := IsMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git", nonAdminUser) | |||
@@ -33,8 +34,8 @@ func TestMigrateWhiteBlocklist(t *testing.T) { | |||
err = IsMigrateURLAllowed("https://gITHUb.com/go-gitea/gitea.git", nonAdminUser) | |||
assert.NoError(t, err) | |||
setting.Migrations.AllowedDomains = []string{} | |||
setting.Migrations.BlockedDomains = []string{"github.com"} | |||
setting.Migrations.AllowedDomains = "" | |||
setting.Migrations.BlockedDomains = "github.com" | |||
assert.NoError(t, Init()) | |||
err = IsMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git", nonAdminUser) | |||
@@ -47,6 +48,7 @@ func TestMigrateWhiteBlocklist(t *testing.T) { | |||
assert.Error(t, err) | |||
setting.Migrations.AllowLocalNetworks = true | |||
assert.NoError(t, Init()) | |||
err = IsMigrateURLAllowed("https://10.0.0.1/go-gitea/gitea.git", nonAdminUser) | |||
assert.NoError(t, err) | |||
@@ -261,8 +261,9 @@ func runSync(ctx context.Context, m *models.Mirror) ([]*mirrorSyncResult, bool) | |||
if m.LFS && setting.LFS.StartServer { | |||
log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo) | |||
ep := lfs.DetermineEndpoint(remoteAddr.String(), m.LFSEndpoint) | |||
if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, ep, false); err != nil { | |||
endpoint := lfs.DetermineEndpoint(remoteAddr.String(), m.LFSEndpoint) | |||
lfsClient := lfs.NewClient(endpoint, nil) | |||
if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, lfsClient); err != nil { | |||
log.Error("Failed to synchronize LFS objects for repository: %v", err) | |||
} | |||
} |
@@ -8,7 +8,6 @@ import ( | |||
"context" | |||
"errors" | |||
"io" | |||
"net/url" | |||
"regexp" | |||
"time" | |||
@@ -133,8 +132,9 @@ func runPushSync(ctx context.Context, m *models.PushMirror) error { | |||
} | |||
defer gitRepo.Close() | |||
ep := lfs.DetermineEndpoint(remoteAddr.String(), "") | |||
if err := pushAllLFSObjects(ctx, gitRepo, ep, false); err != nil { | |||
endpoint := lfs.DetermineEndpoint(remoteAddr.String(), "") | |||
lfsClient := lfs.NewClient(endpoint, nil) | |||
if err := pushAllLFSObjects(ctx, gitRepo, lfsClient); err != nil { | |||
return util.NewURLSanitizedError(err, remoteAddr, true) | |||
} | |||
} | |||
@@ -176,8 +176,7 @@ func runPushSync(ctx context.Context, m *models.PushMirror) error { | |||
return nil | |||
} | |||
func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *url.URL, skipTLSVerify bool) error { | |||
client := lfs.NewClient(endpoint, skipTLSVerify) | |||
func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, lfsClient lfs.Client) error { | |||
contentStore := lfs.NewContentStore() | |||
pointerChan := make(chan lfs.PointerBlob) | |||
@@ -185,7 +184,7 @@ func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *u | |||
go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan) | |||
uploadObjects := func(pointers []lfs.Pointer) error { | |||
err := client.Upload(ctx, pointers, func(p lfs.Pointer, objectError error) (io.ReadCloser, error) { | |||
err := lfsClient.Upload(ctx, pointers, func(p lfs.Pointer, objectError error) (io.ReadCloser, error) { | |||
if objectError != nil { | |||
return nil, objectError | |||
} | |||
@@ -219,7 +218,7 @@ func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *u | |||
} | |||
batch = append(batch, pointerBlob.Pointer) | |||
if len(batch) >= client.BatchSize() { | |||
if len(batch) >= lfsClient.BatchSize() { | |||
if err := uploadObjects(batch); err != nil { | |||
return err | |||
} |
@@ -13,17 +13,16 @@ import ( | |||
"encoding/hex" | |||
"fmt" | |||
"io" | |||
"net" | |||
"net/http" | |||
"net/url" | |||
"strconv" | |||
"strings" | |||
"sync" | |||
"syscall" | |||
"time" | |||
webhook_model "code.gitea.io/gitea/models/webhook" | |||
"code.gitea.io/gitea/modules/graceful" | |||
"code.gitea.io/gitea/modules/hostmatcher" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/proxy" | |||
"code.gitea.io/gitea/modules/setting" | |||
@@ -31,8 +30,6 @@ import ( | |||
"github.com/gobwas/glob" | |||
) | |||
var contextKeyWebhookRequest interface{} = "contextKeyWebhookRequest" | |||
// Deliver deliver hook task | |||
func Deliver(t *webhook_model.HookTask) error { | |||
w, err := webhook_model.GetWebhookByID(t.HookID) | |||
@@ -98,10 +95,10 @@ func Deliver(t *webhook_model.HookTask) error { | |||
return err | |||
} | |||
default: | |||
return fmt.Errorf("Invalid http method for webhook: [%d] %v", t.ID, w.HTTPMethod) | |||
return fmt.Errorf("invalid http method for webhook: [%d] %v", t.ID, w.HTTPMethod) | |||
} | |||
default: | |||
return fmt.Errorf("Invalid http method for webhook: [%d] %v", t.ID, w.HTTPMethod) | |||
return fmt.Errorf("invalid http method for webhook: [%d] %v", t.ID, w.HTTPMethod) | |||
} | |||
var signatureSHA1 string | |||
@@ -172,10 +169,10 @@ func Deliver(t *webhook_model.HookTask) error { | |||
}() | |||
if setting.DisableWebhooks { | |||
return fmt.Errorf("Webhook task skipped (webhooks disabled): [%d]", t.ID) | |||
return fmt.Errorf("webhook task skipped (webhooks disabled): [%d]", t.ID) | |||
} | |||
resp, err := webhookHTTPClient.Do(req.WithContext(context.WithValue(req.Context(), contextKeyWebhookRequest, req))) | |||
resp, err := webhookHTTPClient.Do(req.WithContext(graceful.GetManager().ShutdownContext())) | |||
if err != nil { | |||
t.ResponseInfo.Body = fmt.Sprintf("Delivery: %v", err) | |||
return err | |||
@@ -296,29 +293,18 @@ func webhookProxy() func(req *http.Request) (*url.URL, error) { | |||
func InitDeliverHooks() { | |||
timeout := time.Duration(setting.Webhook.DeliverTimeout) * time.Second | |||
allowedHostListValue := setting.Webhook.AllowedHostList | |||
if allowedHostListValue == "" { | |||
allowedHostListValue = hostmatcher.MatchBuiltinExternal | |||
} | |||
allowedHostMatcher := hostmatcher.ParseHostMatchList("webhook.ALLOWED_HOST_LIST", allowedHostListValue) | |||
webhookHTTPClient = &http.Client{ | |||
Timeout: timeout, | |||
Transport: &http.Transport{ | |||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Webhook.SkipTLSVerify}, | |||
Proxy: webhookProxy(), | |||
DialContext: func(ctx context.Context, network, addrOrHost string) (net.Conn, error) { | |||
dialer := net.Dialer{ | |||
Timeout: timeout, | |||
Control: func(network, ipAddr string, c syscall.RawConn) error { | |||
// in Control func, the addr was already resolved to IP:PORT format, there is no cost to do ResolveTCPAddr here | |||
tcpAddr, err := net.ResolveTCPAddr(network, ipAddr) | |||
req := ctx.Value(contextKeyWebhookRequest).(*http.Request) | |||
if err != nil { | |||
return fmt.Errorf("webhook can only call HTTP servers via TCP, deny '%s(%s:%s)', err=%v", req.Host, network, ipAddr, err) | |||
} | |||
if !setting.Webhook.AllowedHostList.MatchesHostOrIP(req.Host, tcpAddr.IP) { | |||
return fmt.Errorf("webhook can only call allowed HTTP servers (check your webhook.ALLOWED_HOST_LIST setting), deny '%s(%s)'", req.Host, ipAddr) | |||
} | |||
return nil | |||
}, | |||
} | |||
return dialer.DialContext(ctx, network, addrOrHost) | |||
}, | |||
DialContext: hostmatcher.NewDialContext("webhook", allowedHostMatcher, nil), | |||
}, | |||
} | |||