* add black list and white list support for migrating repositories * fix fmt * fix lint * fix vendor * fix modules.txt * clean diff * specify log message * use blocklist/allowlist * allways use lowercase to match url * Apply allow/block * Settings: use existing "migrations" section * convert domains lower case * dont store unused value * Block private addresses for migration by default * fix lint * use proposed-upstream func to detect private IP addr * a nit * add own error for blocked migration, add tests, imprufe api * fix test * fix-if-localhost-is-ipv4 * rename error & error message * rename setting options * Apply suggestions from code review Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com> Co-authored-by: zeripath <art27@cantab.net> Co-authored-by: techknowlogick <techknowlogick@gitea.io>tags/v1.15.0-dev
@@ -1187,6 +1187,14 @@ QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0" | |||
MAX_ATTEMPTS = 3 | |||
; Backoff time per http/https request retry (seconds) | |||
RETRY_BACKOFF = 3 | |||
; Allowed domains for migrating, default is blank. Blank means everything will be allowed. | |||
; Multiple domains could be separated by commas. | |||
ALLOWED_DOMAINS = | |||
; Blocklist for migrating, default is blank. Multiple domains could be separated by commas. | |||
; When ALLOWED_DOMAINS is not blank, this option will be ignored. | |||
BLOCKED_DOMAINS = | |||
; Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291 (false by default) | |||
ALLOW_LOCALNETWORKS = false | |||
; default storage for attachments, lfs and avatars | |||
[storage] |
@@ -846,6 +846,9 @@ Task queue configuration has been moved to `queue.task`. However, the below conf | |||
- `MAX_ATTEMPTS`: **3**: Max attempts per http/https request on migrations. | |||
- `RETRY_BACKOFF`: **3**: Backoff time per http/https request retry (seconds) | |||
- `ALLOWED_DOMAINS`: **\<empty\>**: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas. | |||
- `BLOCKED_DOMAINS`: **\<empty\>**: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWED_DOMAINS` is not blank, this option will be ignored. | |||
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291 | |||
## Mirror (`mirror`) | |||
@@ -313,6 +313,9 @@ IS_INPUT_FILE = false | |||
- `MAX_ATTEMPTS`: **3**: 在迁移过程中的 http/https 请求重试次数。 | |||
- `RETRY_BACKOFF`: **3**: 等待下一次重试的时间,单位秒。 | |||
- `ALLOWED_DOMAINS`: **\<empty\>**: 迁移仓库的域名白名单,默认为空,表示允许从任意域名迁移仓库,多个域名用逗号分隔。 | |||
- `BLOCKED_DOMAINS`: **\<empty\>**: 迁移仓库的域名黑名单,默认为空,多个域名用逗号分隔。如果 `ALLOWED_DOMAINS` 不为空,此选项将会被忽略。 | |||
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918 | |||
## LFS (`lfs`) | |||
@@ -309,6 +309,8 @@ func TestAPIRepoMigrate(t *testing.T) { | |||
{ctxUserID: 2, userID: 1, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-bad", expectedStatus: http.StatusForbidden}, | |||
{ctxUserID: 2, userID: 3, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-org", expectedStatus: http.StatusCreated}, | |||
{ctxUserID: 2, userID: 6, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-bad-org", expectedStatus: http.StatusForbidden}, | |||
{ctxUserID: 2, userID: 3, cloneURL: "https://localhost:3000/user/test_repo.git", repoName: "local-ip", expectedStatus: http.StatusUnprocessableEntity}, | |||
{ctxUserID: 2, userID: 3, cloneURL: "https://10.0.0.1/user/test_repo.git", repoName: "private-ip", expectedStatus: http.StatusUnprocessableEntity}, | |||
} | |||
defer prepareTestEnv(t)() | |||
@@ -325,8 +327,16 @@ func TestAPIRepoMigrate(t *testing.T) { | |||
if resp.Code == http.StatusUnprocessableEntity { | |||
respJSON := map[string]string{} | |||
DecodeJSON(t, resp, &respJSON) | |||
if assert.Equal(t, "Remote visit addressed rate limitation.", respJSON["message"]) { | |||
switch respJSON["message"] { | |||
case "Remote visit addressed rate limitation.": | |||
t.Log("test hit github rate limitation") | |||
case "migrate from '10.0.0.1' is not allowed: the host resolve to a private ip address '10.0.0.1'": | |||
assert.EqualValues(t, "private-ip", testCase.repoName) | |||
case "migrate from 'localhost:3000' is not allowed: the host resolve to a private ip address '::1'", | |||
"migrate from 'localhost:3000' is not allowed: the host resolve to a private ip address '127.0.0.1'": | |||
assert.EqualValues(t, "local-ip", testCase.repoName) | |||
default: | |||
t.Errorf("unexpected error '%v' on url '%s'", respJSON["message"], testCase.cloneURL) | |||
} | |||
} else { | |||
assert.EqualValues(t, testCase.expectedStatus, resp.Code) |
@@ -1019,6 +1019,29 @@ func IsErrWontSign(err error) bool { | |||
return ok | |||
} | |||
// ErrMigrationNotAllowed explains why a migration from an url is not allowed | |||
type ErrMigrationNotAllowed struct { | |||
Host string | |||
NotResolvedIP bool | |||
PrivateNet string | |||
} | |||
func (e *ErrMigrationNotAllowed) Error() string { | |||
if e.NotResolvedIP { | |||
return fmt.Sprintf("migrate from '%s' is not allowed: unknown hostname", e.Host) | |||
} | |||
if len(e.PrivateNet) != 0 { | |||
return fmt.Sprintf("migrate from '%s' is not allowed: the host resolve to a private ip address '%s'", e.Host, e.PrivateNet) | |||
} | |||
return fmt.Sprintf("migrate from '%s is not allowed'", e.Host) | |||
} | |||
// IsErrMigrationNotAllowed checks if an error is a ErrMigrationNotAllowed | |||
func IsErrMigrationNotAllowed(err error) bool { | |||
_, ok := err.(*ErrMigrationNotAllowed) | |||
return ok | |||
} | |||
// __________ .__ | |||
// \______ \____________ ____ ____ | |__ | |||
// | | _/\_ __ \__ \ / \_/ ___\| | \ |
@@ -0,0 +1,46 @@ | |||
// Copyright 2019 The Gitea Authors. All rights reserved. | |||
// Use of this source code is governed by a MIT-style | |||
// license that can be found in the LICENSE file. | |||
package matchlist | |||
import ( | |||
"strings" | |||
"github.com/gobwas/glob" | |||
) | |||
// Matchlist represents a block or allow list | |||
type Matchlist struct { | |||
ruleGlobs []glob.Glob | |||
} | |||
// NewMatchlist creates a new block or allow list | |||
func NewMatchlist(rules ...string) (*Matchlist, error) { | |||
for i := range rules { | |||
rules[i] = strings.ToLower(rules[i]) | |||
} | |||
list := Matchlist{ | |||
ruleGlobs: make([]glob.Glob, 0, len(rules)), | |||
} | |||
for _, rule := range rules { | |||
rg, err := glob.Compile(rule) | |||
if err != nil { | |||
return nil, err | |||
} | |||
list.ruleGlobs = append(list.ruleGlobs, rg) | |||
} | |||
return &list, nil | |||
} | |||
// Match will matches | |||
func (b *Matchlist) Match(u string) bool { | |||
for _, r := range b.ruleGlobs { | |||
if r.Match(u) { | |||
return true | |||
} | |||
} | |||
return false | |||
} |
@@ -8,9 +8,13 @@ package migrations | |||
import ( | |||
"context" | |||
"fmt" | |||
"net" | |||
"net/url" | |||
"strings" | |||
"code.gitea.io/gitea/models" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/matchlist" | |||
"code.gitea.io/gitea/modules/migrations/base" | |||
"code.gitea.io/gitea/modules/setting" | |||
) | |||
@@ -20,6 +24,9 @@ type MigrateOptions = base.MigrateOptions | |||
var ( | |||
factories []base.DownloaderFactory | |||
allowList *matchlist.Matchlist | |||
blockList *matchlist.Matchlist | |||
) | |||
// RegisterDownloaderFactory registers a downloader factory | |||
@@ -27,12 +34,49 @@ func RegisterDownloaderFactory(factory base.DownloaderFactory) { | |||
factories = append(factories, factory) | |||
} | |||
func isMigrateURLAllowed(remoteURL string) error { | |||
u, err := url.Parse(strings.ToLower(remoteURL)) | |||
if err != nil { | |||
return err | |||
} | |||
if strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https") { | |||
if len(setting.Migrations.AllowedDomains) > 0 { | |||
if !allowList.Match(u.Host) { | |||
return &models.ErrMigrationNotAllowed{Host: u.Host} | |||
} | |||
} else { | |||
if blockList.Match(u.Host) { | |||
return &models.ErrMigrationNotAllowed{Host: u.Host} | |||
} | |||
} | |||
} | |||
if !setting.Migrations.AllowLocalNetworks { | |||
addrList, err := net.LookupIP(strings.Split(u.Host, ":")[0]) | |||
if err != nil { | |||
return &models.ErrMigrationNotAllowed{Host: u.Host, NotResolvedIP: true} | |||
} | |||
for _, addr := range addrList { | |||
if isIPPrivate(addr) || !addr.IsGlobalUnicast() { | |||
return &models.ErrMigrationNotAllowed{Host: u.Host, PrivateNet: addr.String()} | |||
} | |||
} | |||
} | |||
return nil | |||
} | |||
// MigrateRepository migrate repository according MigrateOptions | |||
func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) { | |||
err := isMigrateURLAllowed(opts.CloneAddr) | |||
if err != nil { | |||
return nil, err | |||
} | |||
var ( | |||
downloader base.Downloader | |||
uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) | |||
err error | |||
) | |||
for _, factory := range factories { | |||
@@ -308,3 +352,32 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts | |||
return nil | |||
} | |||
// Init migrations service | |||
func Init() error { | |||
var err error | |||
allowList, err = matchlist.NewMatchlist(setting.Migrations.AllowedDomains...) | |||
if err != nil { | |||
return fmt.Errorf("init migration allowList domains failed: %v", err) | |||
} | |||
blockList, err = matchlist.NewMatchlist(setting.Migrations.BlockedDomains...) | |||
if err != nil { | |||
return fmt.Errorf("init migration blockList domains failed: %v", err) | |||
} | |||
return nil | |||
} | |||
// isIPPrivate reports whether ip is a private address, according to | |||
// RFC 1918 (IPv4 addresses) and RFC 4193 (IPv6 addresses). | |||
// from https://github.com/golang/go/pull/42793 | |||
// TODO remove if https://github.com/golang/go/issues/29146 got resolved | |||
func isIPPrivate(ip net.IP) bool { | |||
if ip4 := ip.To4(); ip4 != nil { | |||
return ip4[0] == 10 || | |||
(ip4[0] == 172 && ip4[1]&0xf0 == 16) || | |||
(ip4[0] == 192 && ip4[1] == 168) | |||
} | |||
return len(ip) == net.IPv6len && ip[0]&0xfe == 0xfc | |||
} |
@@ -0,0 +1,34 @@ | |||
// Copyright 2019 The Gitea Authors. All rights reserved. | |||
// Use of this source code is governed by a MIT-style | |||
// license that can be found in the LICENSE file. | |||
package migrations | |||
import ( | |||
"testing" | |||
"code.gitea.io/gitea/modules/setting" | |||
"github.com/stretchr/testify/assert" | |||
) | |||
func TestMigrateWhiteBlocklist(t *testing.T) { | |||
setting.Migrations.AllowedDomains = []string{"github.com"} | |||
assert.NoError(t, Init()) | |||
err := isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git") | |||
assert.Error(t, err) | |||
err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git") | |||
assert.NoError(t, err) | |||
setting.Migrations.AllowedDomains = []string{} | |||
setting.Migrations.BlockedDomains = []string{"github.com"} | |||
assert.NoError(t, Init()) | |||
err = isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git") | |||
assert.NoError(t, err) | |||
err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git") | |||
assert.Error(t, err) | |||
} |
@@ -4,11 +4,18 @@ | |||
package setting | |||
import ( | |||
"strings" | |||
) | |||
var ( | |||
// Migrations settings | |||
Migrations = struct { | |||
MaxAttempts int | |||
RetryBackoff int | |||
MaxAttempts int | |||
RetryBackoff int | |||
AllowedDomains []string | |||
BlockedDomains []string | |||
AllowLocalNetworks bool | |||
}{ | |||
MaxAttempts: 3, | |||
RetryBackoff: 3, | |||
@@ -19,4 +26,15 @@ func newMigrationsService() { | |||
sec := Cfg.Section("migrations") | |||
Migrations.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Migrations.MaxAttempts) | |||
Migrations.RetryBackoff = sec.Key("RETRY_BACKOFF").MustInt(Migrations.RetryBackoff) | |||
Migrations.AllowedDomains = sec.Key("ALLOWED_DOMAINS").Strings(",") | |||
for i := range Migrations.AllowedDomains { | |||
Migrations.AllowedDomains[i] = strings.ToLower(Migrations.AllowedDomains[i]) | |||
} | |||
Migrations.BlockedDomains = sec.Key("BLOCKED_DOMAINS").Strings(",") | |||
for i := range Migrations.BlockedDomains { | |||
Migrations.BlockedDomains[i] = strings.ToLower(Migrations.BlockedDomains[i]) | |||
} | |||
Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false) | |||
} |
@@ -212,6 +212,8 @@ func handleMigrateError(ctx *context.APIContext, repoOwner *models.User, remoteA | |||
ctx.Error(http.StatusUnprocessableEntity, "", fmt.Sprintf("The username '%s' contains invalid characters.", err.(models.ErrNameCharsNotAllowed).Name)) | |||
case models.IsErrNamePatternNotAllowed(err): | |||
ctx.Error(http.StatusUnprocessableEntity, "", fmt.Sprintf("The pattern '%s' is not allowed in a username.", err.(models.ErrNamePatternNotAllowed).Pattern)) | |||
case models.IsErrMigrationNotAllowed(err): | |||
ctx.Error(http.StatusUnprocessableEntity, "", err) | |||
default: | |||
err = util.URLSanitizedError(err, remoteAddr) | |||
if strings.Contains(err.Error(), "Authentication failed") || |
@@ -24,6 +24,7 @@ import ( | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/markup" | |||
"code.gitea.io/gitea/modules/markup/external" | |||
repo_migrations "code.gitea.io/gitea/modules/migrations" | |||
"code.gitea.io/gitea/modules/notification" | |||
"code.gitea.io/gitea/modules/options" | |||
"code.gitea.io/gitea/modules/setting" | |||
@@ -201,6 +202,9 @@ func GlobalInit(ctx context.Context) { | |||
if err := task.Init(); err != nil { | |||
log.Fatal("Failed to initialize task scheduler: %v", err) | |||
} | |||
if err := repo_migrations.Init(); err != nil { | |||
log.Fatal("Failed to initialize repository migrations: %v", err) | |||
} | |||
eventsource.GetManager().Init() | |||
if setting.EnableSQLite3 { |