* add black list and white list support for migrating repositories * fix fmt * fix lint * fix vendor * fix modules.txt * clean diff * specify log message * use blocklist/allowlist * allways use lowercase to match url * Apply allow/block * Settings: use existing "migrations" section * convert domains lower case * dont store unused value * Block private addresses for migration by default * fix lint * use proposed-upstream func to detect private IP addr * a nit * add own error for blocked migration, add tests, imprufe api * fix test * fix-if-localhost-is-ipv4 * rename error & error message * rename setting options * Apply suggestions from code review Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com> Co-authored-by: zeripath <art27@cantab.net> Co-authored-by: techknowlogick <techknowlogick@gitea.io>tags/v1.15.0-dev
MAX_ATTEMPTS = 3 | MAX_ATTEMPTS = 3 | ||||
; Backoff time per http/https request retry (seconds) | ; Backoff time per http/https request retry (seconds) | ||||
RETRY_BACKOFF = 3 | RETRY_BACKOFF = 3 | ||||
; Allowed domains for migrating, default is blank. Blank means everything will be allowed. | |||||
; Multiple domains could be separated by commas. | |||||
ALLOWED_DOMAINS = | |||||
; Blocklist for migrating, default is blank. Multiple domains could be separated by commas. | |||||
; When ALLOWED_DOMAINS is not blank, this option will be ignored. | |||||
BLOCKED_DOMAINS = | |||||
; Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291 (false by default) | |||||
ALLOW_LOCALNETWORKS = false | |||||
; default storage for attachments, lfs and avatars | ; default storage for attachments, lfs and avatars | ||||
[storage] | [storage] |
- `MAX_ATTEMPTS`: **3**: Max attempts per http/https request on migrations. | - `MAX_ATTEMPTS`: **3**: Max attempts per http/https request on migrations. | ||||
- `RETRY_BACKOFF`: **3**: Backoff time per http/https request retry (seconds) | - `RETRY_BACKOFF`: **3**: Backoff time per http/https request retry (seconds) | ||||
- `ALLOWED_DOMAINS`: **\<empty\>**: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas. | |||||
- `BLOCKED_DOMAINS`: **\<empty\>**: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWED_DOMAINS` is not blank, this option will be ignored. | |||||
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291 | |||||
## Mirror (`mirror`) | ## Mirror (`mirror`) | ||||
- `MAX_ATTEMPTS`: **3**: 在迁移过程中的 http/https 请求重试次数。 | - `MAX_ATTEMPTS`: **3**: 在迁移过程中的 http/https 请求重试次数。 | ||||
- `RETRY_BACKOFF`: **3**: 等待下一次重试的时间,单位秒。 | - `RETRY_BACKOFF`: **3**: 等待下一次重试的时间,单位秒。 | ||||
- `ALLOWED_DOMAINS`: **\<empty\>**: 迁移仓库的域名白名单,默认为空,表示允许从任意域名迁移仓库,多个域名用逗号分隔。 | |||||
- `BLOCKED_DOMAINS`: **\<empty\>**: 迁移仓库的域名黑名单,默认为空,多个域名用逗号分隔。如果 `ALLOWED_DOMAINS` 不为空,此选项将会被忽略。 | |||||
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918 | |||||
## LFS (`lfs`) | ## LFS (`lfs`) | ||||
{ctxUserID: 2, userID: 1, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-bad", expectedStatus: http.StatusForbidden}, | {ctxUserID: 2, userID: 1, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-bad", expectedStatus: http.StatusForbidden}, | ||||
{ctxUserID: 2, userID: 3, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-org", expectedStatus: http.StatusCreated}, | {ctxUserID: 2, userID: 3, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-org", expectedStatus: http.StatusCreated}, | ||||
{ctxUserID: 2, userID: 6, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-bad-org", expectedStatus: http.StatusForbidden}, | {ctxUserID: 2, userID: 6, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-bad-org", expectedStatus: http.StatusForbidden}, | ||||
{ctxUserID: 2, userID: 3, cloneURL: "https://localhost:3000/user/test_repo.git", repoName: "local-ip", expectedStatus: http.StatusUnprocessableEntity}, | |||||
{ctxUserID: 2, userID: 3, cloneURL: "https://10.0.0.1/user/test_repo.git", repoName: "private-ip", expectedStatus: http.StatusUnprocessableEntity}, | |||||
} | } | ||||
defer prepareTestEnv(t)() | defer prepareTestEnv(t)() | ||||
if resp.Code == http.StatusUnprocessableEntity { | if resp.Code == http.StatusUnprocessableEntity { | ||||
respJSON := map[string]string{} | respJSON := map[string]string{} | ||||
DecodeJSON(t, resp, &respJSON) | DecodeJSON(t, resp, &respJSON) | ||||
if assert.Equal(t, "Remote visit addressed rate limitation.", respJSON["message"]) { | |||||
switch respJSON["message"] { | |||||
case "Remote visit addressed rate limitation.": | |||||
t.Log("test hit github rate limitation") | t.Log("test hit github rate limitation") | ||||
case "migrate from '10.0.0.1' is not allowed: the host resolve to a private ip address '10.0.0.1'": | |||||
assert.EqualValues(t, "private-ip", testCase.repoName) | |||||
case "migrate from 'localhost:3000' is not allowed: the host resolve to a private ip address '::1'", | |||||
"migrate from 'localhost:3000' is not allowed: the host resolve to a private ip address '127.0.0.1'": | |||||
assert.EqualValues(t, "local-ip", testCase.repoName) | |||||
default: | |||||
t.Errorf("unexpected error '%v' on url '%s'", respJSON["message"], testCase.cloneURL) | |||||
} | } | ||||
} else { | } else { | ||||
assert.EqualValues(t, testCase.expectedStatus, resp.Code) | assert.EqualValues(t, testCase.expectedStatus, resp.Code) |
return ok | return ok | ||||
} | } | ||||
// ErrMigrationNotAllowed explains why a migration from an url is not allowed | |||||
type ErrMigrationNotAllowed struct { | |||||
Host string | |||||
NotResolvedIP bool | |||||
PrivateNet string | |||||
} | |||||
func (e *ErrMigrationNotAllowed) Error() string { | |||||
if e.NotResolvedIP { | |||||
return fmt.Sprintf("migrate from '%s' is not allowed: unknown hostname", e.Host) | |||||
} | |||||
if len(e.PrivateNet) != 0 { | |||||
return fmt.Sprintf("migrate from '%s' is not allowed: the host resolve to a private ip address '%s'", e.Host, e.PrivateNet) | |||||
} | |||||
return fmt.Sprintf("migrate from '%s is not allowed'", e.Host) | |||||
} | |||||
// IsErrMigrationNotAllowed checks if an error is a ErrMigrationNotAllowed | |||||
func IsErrMigrationNotAllowed(err error) bool { | |||||
_, ok := err.(*ErrMigrationNotAllowed) | |||||
return ok | |||||
} | |||||
// __________ .__ | // __________ .__ | ||||
// \______ \____________ ____ ____ | |__ | // \______ \____________ ____ ____ | |__ | ||||
// | | _/\_ __ \__ \ / \_/ ___\| | \ | // | | _/\_ __ \__ \ / \_/ ___\| | \ |
// Copyright 2019 The Gitea Authors. All rights reserved. | |||||
// Use of this source code is governed by a MIT-style | |||||
// license that can be found in the LICENSE file. | |||||
package matchlist | |||||
import ( | |||||
"strings" | |||||
"github.com/gobwas/glob" | |||||
) | |||||
// Matchlist represents a block or allow list | |||||
type Matchlist struct { | |||||
ruleGlobs []glob.Glob | |||||
} | |||||
// NewMatchlist creates a new block or allow list | |||||
func NewMatchlist(rules ...string) (*Matchlist, error) { | |||||
for i := range rules { | |||||
rules[i] = strings.ToLower(rules[i]) | |||||
} | |||||
list := Matchlist{ | |||||
ruleGlobs: make([]glob.Glob, 0, len(rules)), | |||||
} | |||||
for _, rule := range rules { | |||||
rg, err := glob.Compile(rule) | |||||
if err != nil { | |||||
return nil, err | |||||
} | |||||
list.ruleGlobs = append(list.ruleGlobs, rg) | |||||
} | |||||
return &list, nil | |||||
} | |||||
// Match will matches | |||||
func (b *Matchlist) Match(u string) bool { | |||||
for _, r := range b.ruleGlobs { | |||||
if r.Match(u) { | |||||
return true | |||||
} | |||||
} | |||||
return false | |||||
} |
import ( | import ( | ||||
"context" | "context" | ||||
"fmt" | "fmt" | ||||
"net" | |||||
"net/url" | |||||
"strings" | |||||
"code.gitea.io/gitea/models" | "code.gitea.io/gitea/models" | ||||
"code.gitea.io/gitea/modules/log" | "code.gitea.io/gitea/modules/log" | ||||
"code.gitea.io/gitea/modules/matchlist" | |||||
"code.gitea.io/gitea/modules/migrations/base" | "code.gitea.io/gitea/modules/migrations/base" | ||||
"code.gitea.io/gitea/modules/setting" | "code.gitea.io/gitea/modules/setting" | ||||
) | ) | ||||
var ( | var ( | ||||
factories []base.DownloaderFactory | factories []base.DownloaderFactory | ||||
allowList *matchlist.Matchlist | |||||
blockList *matchlist.Matchlist | |||||
) | ) | ||||
// RegisterDownloaderFactory registers a downloader factory | // RegisterDownloaderFactory registers a downloader factory | ||||
factories = append(factories, factory) | factories = append(factories, factory) | ||||
} | } | ||||
func isMigrateURLAllowed(remoteURL string) error { | |||||
u, err := url.Parse(strings.ToLower(remoteURL)) | |||||
if err != nil { | |||||
return err | |||||
} | |||||
if strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https") { | |||||
if len(setting.Migrations.AllowedDomains) > 0 { | |||||
if !allowList.Match(u.Host) { | |||||
return &models.ErrMigrationNotAllowed{Host: u.Host} | |||||
} | |||||
} else { | |||||
if blockList.Match(u.Host) { | |||||
return &models.ErrMigrationNotAllowed{Host: u.Host} | |||||
} | |||||
} | |||||
} | |||||
if !setting.Migrations.AllowLocalNetworks { | |||||
addrList, err := net.LookupIP(strings.Split(u.Host, ":")[0]) | |||||
if err != nil { | |||||
return &models.ErrMigrationNotAllowed{Host: u.Host, NotResolvedIP: true} | |||||
} | |||||
for _, addr := range addrList { | |||||
if isIPPrivate(addr) || !addr.IsGlobalUnicast() { | |||||
return &models.ErrMigrationNotAllowed{Host: u.Host, PrivateNet: addr.String()} | |||||
} | |||||
} | |||||
} | |||||
return nil | |||||
} | |||||
// MigrateRepository migrate repository according MigrateOptions | // MigrateRepository migrate repository according MigrateOptions | ||||
func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) { | func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) { | ||||
err := isMigrateURLAllowed(opts.CloneAddr) | |||||
if err != nil { | |||||
return nil, err | |||||
} | |||||
var ( | var ( | ||||
downloader base.Downloader | downloader base.Downloader | ||||
uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) | uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) | ||||
err error | |||||
) | ) | ||||
for _, factory := range factories { | for _, factory := range factories { | ||||
return nil | return nil | ||||
} | } | ||||
// Init migrations service | |||||
func Init() error { | |||||
var err error | |||||
allowList, err = matchlist.NewMatchlist(setting.Migrations.AllowedDomains...) | |||||
if err != nil { | |||||
return fmt.Errorf("init migration allowList domains failed: %v", err) | |||||
} | |||||
blockList, err = matchlist.NewMatchlist(setting.Migrations.BlockedDomains...) | |||||
if err != nil { | |||||
return fmt.Errorf("init migration blockList domains failed: %v", err) | |||||
} | |||||
return nil | |||||
} | |||||
// isIPPrivate reports whether ip is a private address, according to | |||||
// RFC 1918 (IPv4 addresses) and RFC 4193 (IPv6 addresses). | |||||
// from https://github.com/golang/go/pull/42793 | |||||
// TODO remove if https://github.com/golang/go/issues/29146 got resolved | |||||
func isIPPrivate(ip net.IP) bool { | |||||
if ip4 := ip.To4(); ip4 != nil { | |||||
return ip4[0] == 10 || | |||||
(ip4[0] == 172 && ip4[1]&0xf0 == 16) || | |||||
(ip4[0] == 192 && ip4[1] == 168) | |||||
} | |||||
return len(ip) == net.IPv6len && ip[0]&0xfe == 0xfc | |||||
} |
// Copyright 2019 The Gitea Authors. All rights reserved. | |||||
// Use of this source code is governed by a MIT-style | |||||
// license that can be found in the LICENSE file. | |||||
package migrations | |||||
import ( | |||||
"testing" | |||||
"code.gitea.io/gitea/modules/setting" | |||||
"github.com/stretchr/testify/assert" | |||||
) | |||||
func TestMigrateWhiteBlocklist(t *testing.T) { | |||||
setting.Migrations.AllowedDomains = []string{"github.com"} | |||||
assert.NoError(t, Init()) | |||||
err := isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git") | |||||
assert.Error(t, err) | |||||
err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git") | |||||
assert.NoError(t, err) | |||||
setting.Migrations.AllowedDomains = []string{} | |||||
setting.Migrations.BlockedDomains = []string{"github.com"} | |||||
assert.NoError(t, Init()) | |||||
err = isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git") | |||||
assert.NoError(t, err) | |||||
err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git") | |||||
assert.Error(t, err) | |||||
} |
package setting | package setting | ||||
import ( | |||||
"strings" | |||||
) | |||||
var ( | var ( | ||||
// Migrations settings | // Migrations settings | ||||
Migrations = struct { | Migrations = struct { | ||||
MaxAttempts int | |||||
RetryBackoff int | |||||
MaxAttempts int | |||||
RetryBackoff int | |||||
AllowedDomains []string | |||||
BlockedDomains []string | |||||
AllowLocalNetworks bool | |||||
}{ | }{ | ||||
MaxAttempts: 3, | MaxAttempts: 3, | ||||
RetryBackoff: 3, | RetryBackoff: 3, | ||||
sec := Cfg.Section("migrations") | sec := Cfg.Section("migrations") | ||||
Migrations.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Migrations.MaxAttempts) | Migrations.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Migrations.MaxAttempts) | ||||
Migrations.RetryBackoff = sec.Key("RETRY_BACKOFF").MustInt(Migrations.RetryBackoff) | Migrations.RetryBackoff = sec.Key("RETRY_BACKOFF").MustInt(Migrations.RetryBackoff) | ||||
Migrations.AllowedDomains = sec.Key("ALLOWED_DOMAINS").Strings(",") | |||||
for i := range Migrations.AllowedDomains { | |||||
Migrations.AllowedDomains[i] = strings.ToLower(Migrations.AllowedDomains[i]) | |||||
} | |||||
Migrations.BlockedDomains = sec.Key("BLOCKED_DOMAINS").Strings(",") | |||||
for i := range Migrations.BlockedDomains { | |||||
Migrations.BlockedDomains[i] = strings.ToLower(Migrations.BlockedDomains[i]) | |||||
} | |||||
Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false) | |||||
} | } |
ctx.Error(http.StatusUnprocessableEntity, "", fmt.Sprintf("The username '%s' contains invalid characters.", err.(models.ErrNameCharsNotAllowed).Name)) | ctx.Error(http.StatusUnprocessableEntity, "", fmt.Sprintf("The username '%s' contains invalid characters.", err.(models.ErrNameCharsNotAllowed).Name)) | ||||
case models.IsErrNamePatternNotAllowed(err): | case models.IsErrNamePatternNotAllowed(err): | ||||
ctx.Error(http.StatusUnprocessableEntity, "", fmt.Sprintf("The pattern '%s' is not allowed in a username.", err.(models.ErrNamePatternNotAllowed).Pattern)) | ctx.Error(http.StatusUnprocessableEntity, "", fmt.Sprintf("The pattern '%s' is not allowed in a username.", err.(models.ErrNamePatternNotAllowed).Pattern)) | ||||
case models.IsErrMigrationNotAllowed(err): | |||||
ctx.Error(http.StatusUnprocessableEntity, "", err) | |||||
default: | default: | ||||
err = util.URLSanitizedError(err, remoteAddr) | err = util.URLSanitizedError(err, remoteAddr) | ||||
if strings.Contains(err.Error(), "Authentication failed") || | if strings.Contains(err.Error(), "Authentication failed") || |
"code.gitea.io/gitea/modules/log" | "code.gitea.io/gitea/modules/log" | ||||
"code.gitea.io/gitea/modules/markup" | "code.gitea.io/gitea/modules/markup" | ||||
"code.gitea.io/gitea/modules/markup/external" | "code.gitea.io/gitea/modules/markup/external" | ||||
repo_migrations "code.gitea.io/gitea/modules/migrations" | |||||
"code.gitea.io/gitea/modules/notification" | "code.gitea.io/gitea/modules/notification" | ||||
"code.gitea.io/gitea/modules/options" | "code.gitea.io/gitea/modules/options" | ||||
"code.gitea.io/gitea/modules/setting" | "code.gitea.io/gitea/modules/setting" | ||||
if err := task.Init(); err != nil { | if err := task.Init(); err != nil { | ||||
log.Fatal("Failed to initialize task scheduler: %v", err) | log.Fatal("Failed to initialize task scheduler: %v", err) | ||||
} | } | ||||
if err := repo_migrations.Init(); err != nil { | |||||
log.Fatal("Failed to initialize repository migrations: %v", err) | |||||
} | |||||
eventsource.GetManager().Init() | eventsource.GetManager().Init() | ||||
if setting.EnableSQLite3 { | if setting.EnableSQLite3 { |