* Add proxy settings and support for migration and webhook * Fix default value * Add newline for example ini * Add lfs proxy support * Fix lint * Follow @zeripath's review * Fix git clone * Fix test * missgin http requests for proxy * use empty Co-authored-by: zeripath <art27@cantab.net> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: zeripath <art27@cantab.net>tags/v1.16.0-rc1
@@ -2127,3 +2127,11 @@ PATH = | |||
;; | |||
;; Minio enabled ssl only available when STORAGE_TYPE is `minio` | |||
;MINIO_USE_SSL = false | |||
;[proxy] | |||
;; Enable the proxy, all requests to external via HTTP will be affected | |||
;PROXY_ENABLED = false | |||
;; Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy/no_proxy | |||
;PROXY_URL = | |||
;; Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts. | |||
;PROXY_HOSTS = |
@@ -549,8 +549,8 @@ Define allowed algorithms and their minimum key length (use -1 to disable a type | |||
- `DELIVER_TIMEOUT`: **5**: Delivery timeout (sec) for shooting webhooks. | |||
- `SKIP_TLS_VERIFY`: **false**: Allow insecure certification. | |||
- `PAGING_NUM`: **10**: Number of webhook history events that are shown in one page. | |||
- `PROXY_URL`: ****: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy | |||
- `PROXY_HOSTS`: ****: Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts. | |||
- `PROXY_URL`: **\<empty\>**: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy. If not given, will use global proxy setting. | |||
- `PROXY_HOSTS`: **\<empty\>`**: Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts. If not given, will use global proxy setting. | |||
## Mailer (`mailer`) | |||
@@ -950,6 +950,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf | |||
- `ALLOWED_DOMAINS`: **\<empty\>**: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas. | |||
- `BLOCKED_DOMAINS`: **\<empty\>**: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWED_DOMAINS` is not blank, this option will be ignored. | |||
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291 | |||
- `SKIP_TLS_VERIFY`: **false**: Allow skip tls verify | |||
## Mirror (`mirror`) | |||
@@ -1023,6 +1024,19 @@ is `data/repo-archive` and the default of `MINIO_BASE_PATH` is `repo-archive/`. | |||
- `MINIO_BASE_PATH`: **repo-archive/**: Minio base path on the bucket only available when `STORAGE_TYPE` is `minio` | |||
- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio` | |||
## Proxy (`proxy`) | |||
- `PROXY_ENABLED`: **false**: Enable the proxy if true, all requests to external via HTTP will be affected, if false, no proxy will be used even environment http_proxy/https_proxy | |||
- `PROXY_URL`: **\<empty\>**: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy | |||
- `PROXY_HOSTS`: **\<empty\>**: Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts. | |||
i.e. | |||
```ini | |||
PROXY_ENABLED = true | |||
PROXY_URL = socks://127.0.0.1:1080 | |||
PROXY_HOSTS = *.github.com | |||
``` | |||
## Other (`other`) | |||
- `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer. |
@@ -332,6 +332,7 @@ IS_INPUT_FILE = false | |||
- `ALLOWED_DOMAINS`: **\<empty\>**: 迁移仓库的域名白名单,默认为空,表示允许从任意域名迁移仓库,多个域名用逗号分隔。 | |||
- `BLOCKED_DOMAINS`: **\<empty\>**: 迁移仓库的域名黑名单,默认为空,多个域名用逗号分隔。如果 `ALLOWED_DOMAINS` 不为空,此选项将会被忽略。 | |||
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918 | |||
- `SKIP_TLS_VERIFY`: **false**: 允许忽略 TLS 认证 | |||
## LFS (`lfs`) | |||
@@ -397,6 +398,19 @@ Repository archive 的存储配置。 如果 `STORAGE_TYPE` 为空,则此配 | |||
- `MINIO_BASE_PATH`: **repo-archive/**: Minio base path ,仅当 `STORAGE_TYPE` 为 `minio` 时有效。 | |||
- `MINIO_USE_SSL`: **false**: Minio 是否启用 ssl ,仅当 `STORAGE_TYPE` 为 `minio` 时有效。 | |||
## Proxy (`proxy`) | |||
- `PROXY_ENABLED`: **false**: 是否启用全局代理。如果为否,则不使用代理,环境变量中的代理也不使用 | |||
- `PROXY_URL`: **\<empty\>**: 代理服务器地址,支持 http://, https//, socks://,为空则不启用代理而使用环境变量中的 http_proxy/https_proxy | |||
- `PROXY_HOSTS`: **\<empty\>**: 逗号分隔的多个需要代理的网址,支持 * 号匹配符号, ** 表示匹配所有网站 | |||
i.e. | |||
```ini | |||
PROXY_ENABLED = true | |||
PROXY_URL = socks://127.0.0.1:1080 | |||
PROXY_HOSTS = *.github.com | |||
``` | |||
## Other (`other`) | |||
- `SHOW_FOOTER_BRANDING`: 为真则在页面底部显示Gitea的字样。 |
@@ -110,24 +110,47 @@ func (c *Command) RunInDirTimeoutEnvFullPipeline(env []string, timeout time.Dura | |||
// RunInDirTimeoutEnvFullPipelineFunc executes the command in given directory with given timeout, | |||
// it pipes stdout and stderr to given io.Writer and passes in an io.Reader as stdin. Between cmd.Start and cmd.Wait the passed in function is run. | |||
func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time.Duration, dir string, stdout, stderr io.Writer, stdin io.Reader, fn func(context.Context, context.CancelFunc) error) error { | |||
if timeout == -1 { | |||
timeout = defaultCommandExecutionTimeout | |||
return c.RunWithContext(&RunContext{ | |||
Env: env, | |||
Timeout: timeout, | |||
Dir: dir, | |||
Stdout: stdout, | |||
Stderr: stderr, | |||
Stdin: stdin, | |||
PipelineFunc: fn, | |||
}) | |||
} | |||
// RunContext represents parameters to run the command | |||
type RunContext struct { | |||
Env []string | |||
Timeout time.Duration | |||
Dir string | |||
Stdout, Stderr io.Writer | |||
Stdin io.Reader | |||
PipelineFunc func(context.Context, context.CancelFunc) error | |||
} | |||
// RunWithContext run the command with context | |||
func (c *Command) RunWithContext(rc *RunContext) error { | |||
if rc.Timeout == -1 { | |||
rc.Timeout = defaultCommandExecutionTimeout | |||
} | |||
if len(dir) == 0 { | |||
if len(rc.Dir) == 0 { | |||
log.Debug("%s", c) | |||
} else { | |||
log.Debug("%s: %v", dir, c) | |||
log.Debug("%s: %v", rc.Dir, c) | |||
} | |||
ctx, cancel := context.WithTimeout(c.parentContext, timeout) | |||
ctx, cancel := context.WithTimeout(c.parentContext, rc.Timeout) | |||
defer cancel() | |||
cmd := exec.CommandContext(ctx, c.name, c.args...) | |||
if env == nil { | |||
if rc.Env == nil { | |||
cmd.Env = os.Environ() | |||
} else { | |||
cmd.Env = env | |||
cmd.Env = rc.Env | |||
} | |||
cmd.Env = append( | |||
@@ -141,23 +164,23 @@ func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time. | |||
if goVersionLessThan115 { | |||
cmd.Env = append(cmd.Env, "GODEBUG=asyncpreemptoff=1") | |||
} | |||
cmd.Dir = dir | |||
cmd.Stdout = stdout | |||
cmd.Stderr = stderr | |||
cmd.Stdin = stdin | |||
cmd.Dir = rc.Dir | |||
cmd.Stdout = rc.Stdout | |||
cmd.Stderr = rc.Stderr | |||
cmd.Stdin = rc.Stdin | |||
if err := cmd.Start(); err != nil { | |||
return err | |||
} | |||
desc := c.desc | |||
if desc == "" { | |||
desc = fmt.Sprintf("%s %s %s [repo_path: %s]", GitExecutable, c.name, strings.Join(c.args, " "), dir) | |||
desc = fmt.Sprintf("%s %s %s [repo_path: %s]", GitExecutable, c.name, strings.Join(c.args, " "), rc.Dir) | |||
} | |||
pid := process.GetManager().Add(desc, cancel) | |||
defer process.GetManager().Remove(pid) | |||
if fn != nil { | |||
err := fn(ctx, cancel) | |||
if rc.PipelineFunc != nil { | |||
err := rc.PipelineFunc(ctx, cancel) | |||
if err != nil { | |||
cancel() | |||
_ = cmd.Wait() |
@@ -9,11 +9,15 @@ import ( | |||
"bytes" | |||
"context" | |||
"fmt" | |||
"io" | |||
"net/url" | |||
"os" | |||
"path" | |||
"strconv" | |||
"strings" | |||
"time" | |||
"code.gitea.io/gitea/modules/proxy" | |||
) | |||
// GPGSettings represents the default GPG settings for this repository | |||
@@ -99,12 +103,12 @@ type CloneRepoOptions struct { | |||
} | |||
// Clone clones original repository to target path. | |||
func Clone(from, to string, opts CloneRepoOptions) (err error) { | |||
func Clone(from, to string, opts CloneRepoOptions) error { | |||
return CloneWithContext(DefaultContext, from, to, opts) | |||
} | |||
// CloneWithContext clones original repository to target path. | |||
func CloneWithContext(ctx context.Context, from, to string, opts CloneRepoOptions) (err error) { | |||
func CloneWithContext(ctx context.Context, from, to string, opts CloneRepoOptions) error { | |||
cargs := make([]string, len(GlobalCommandArgs)) | |||
copy(cargs, GlobalCommandArgs) | |||
return CloneWithArgs(ctx, from, to, cargs, opts) | |||
@@ -146,8 +150,24 @@ func CloneWithArgs(ctx context.Context, from, to string, args []string, opts Clo | |||
opts.Timeout = -1 | |||
} | |||
_, err = cmd.RunTimeout(opts.Timeout) | |||
return err | |||
var envs = os.Environ() | |||
u, err := url.Parse(from) | |||
if err == nil && (strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https")) { | |||
if proxy.Match(u.Host) { | |||
envs = append(envs, fmt.Sprintf("https_proxy=%s", proxy.GetProxyURL())) | |||
} | |||
} | |||
var stderr = new(bytes.Buffer) | |||
if err = cmd.RunWithContext(&RunContext{ | |||
Timeout: opts.Timeout, | |||
Env: envs, | |||
Stdout: io.Discard, | |||
Stderr: stderr, | |||
}); err != nil { | |||
return ConcatenateError(err, stderr.String()) | |||
} | |||
return nil | |||
} | |||
// PullRemoteOptions options when pull from remote |
@@ -24,9 +24,9 @@ type Client interface { | |||
} | |||
// NewClient creates a LFS client | |||
func NewClient(endpoint *url.URL) Client { | |||
func NewClient(endpoint *url.URL, skipTLSVerify bool) Client { | |||
if endpoint.Scheme == "file" { | |||
return newFilesystemClient(endpoint) | |||
} | |||
return newHTTPClient(endpoint) | |||
return newHTTPClient(endpoint, skipTLSVerify) | |||
} |
@@ -13,10 +13,10 @@ import ( | |||
func TestNewClient(t *testing.T) { | |||
u, _ := url.Parse("file:///test") | |||
c := NewClient(u) | |||
c := NewClient(u, true) | |||
assert.IsType(t, &FilesystemClient{}, c) | |||
u, _ = url.Parse("https://test.com/lfs") | |||
c = NewClient(u) | |||
c = NewClient(u, true) | |||
assert.IsType(t, &HTTPClient{}, c) | |||
} |
@@ -7,6 +7,7 @@ package lfs | |||
import ( | |||
"bytes" | |||
"context" | |||
"crypto/tls" | |||
"errors" | |||
"fmt" | |||
"net/http" | |||
@@ -15,6 +16,7 @@ import ( | |||
"code.gitea.io/gitea/modules/json" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/proxy" | |||
) | |||
const batchSize = 20 | |||
@@ -32,8 +34,13 @@ func (c *HTTPClient) BatchSize() int { | |||
return batchSize | |||
} | |||
func newHTTPClient(endpoint *url.URL) *HTTPClient { | |||
hc := &http.Client{} | |||
func newHTTPClient(endpoint *url.URL, skipTLSVerify bool) *HTTPClient { | |||
hc := &http.Client{ | |||
Transport: &http.Transport{ | |||
TLSClientConfig: &tls.Config{InsecureSkipVerify: skipTLSVerify}, | |||
Proxy: proxy.Proxy(), | |||
}, | |||
} | |||
client := &HTTPClient{ | |||
client: hc, |
@@ -6,6 +6,7 @@ package migrations | |||
import ( | |||
"context" | |||
"crypto/tls" | |||
"errors" | |||
"fmt" | |||
"io" | |||
@@ -17,6 +18,8 @@ import ( | |||
"code.gitea.io/gitea/models" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/migrations/base" | |||
"code.gitea.io/gitea/modules/proxy" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/modules/structs" | |||
gitea_sdk "code.gitea.io/sdk/gitea" | |||
@@ -87,6 +90,12 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo | |||
gitea_sdk.SetToken(token), | |||
gitea_sdk.SetBasicAuth(username, password), | |||
gitea_sdk.SetContext(ctx), | |||
gitea_sdk.SetHTTPClient(&http.Client{ | |||
Transport: &http.Transport{ | |||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}, | |||
Proxy: proxy.Proxy(), | |||
}, | |||
}), | |||
) | |||
if err != nil { | |||
log.Error(fmt.Sprintf("Failed to create NewGiteaDownloader for: %s. Error: %v", baseURL, err)) | |||
@@ -266,6 +275,13 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele | |||
Created: rel.CreatedAt, | |||
} | |||
httpClient := &http.Client{ | |||
Transport: &http.Transport{ | |||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}, | |||
Proxy: proxy.Proxy(), | |||
}, | |||
} | |||
for _, asset := range rel.Attachments { | |||
size := int(asset.Size) | |||
dlCount := int(asset.DownloadCount) | |||
@@ -282,7 +298,11 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele | |||
return nil, err | |||
} | |||
// FIXME: for a private download? | |||
resp, err := http.Get(asset.DownloadURL) | |||
req, err := http.NewRequest("GET", asset.DownloadURL, nil) | |||
if err != nil { | |||
return nil, err | |||
} | |||
resp, err := httpClient.Do(req) | |||
if err != nil { | |||
return nil, err | |||
} |
@@ -7,6 +7,7 @@ package migrations | |||
import ( | |||
"context" | |||
"crypto/tls" | |||
"fmt" | |||
"io" | |||
"net/http" | |||
@@ -17,6 +18,8 @@ import ( | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/migrations/base" | |||
"code.gitea.io/gitea/modules/proxy" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/modules/structs" | |||
"code.gitea.io/gitea/modules/util" | |||
@@ -90,7 +93,7 @@ func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, tok | |||
Transport: &http.Transport{ | |||
Proxy: func(req *http.Request) (*url.URL, error) { | |||
req.SetBasicAuth(userName, password) | |||
return nil, nil | |||
return proxy.Proxy()(req) | |||
}, | |||
}, | |||
} | |||
@@ -269,6 +272,13 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) | |||
r.Published = rel.PublishedAt.Time | |||
} | |||
httpClient := &http.Client{ | |||
Transport: &http.Transport{ | |||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}, | |||
Proxy: proxy.Proxy(), | |||
}, | |||
} | |||
for _, asset := range rel.Assets { | |||
var assetID = *asset.ID // Don't optimize this, for closure we need a local variable | |||
r.Assets = append(r.Assets, &base.ReleaseAsset{ | |||
@@ -295,7 +305,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) | |||
if err != nil { | |||
return nil, err | |||
} | |||
resp, err := http.DefaultClient.Do(req) | |||
resp, err := httpClient.Do(req) | |||
err1 := g.RefreshRate() | |||
if err1 != nil { | |||
log.Error("g.client.RateLimits: %s", err1) |
@@ -6,6 +6,7 @@ package migrations | |||
import ( | |||
"context" | |||
"crypto/tls" | |||
"errors" | |||
"fmt" | |||
"io" | |||
@@ -17,6 +18,8 @@ import ( | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/migrations/base" | |||
"code.gitea.io/gitea/modules/proxy" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/modules/structs" | |||
"github.com/xanzy/go-gitlab" | |||
@@ -77,7 +80,12 @@ type GitlabDownloader struct { | |||
// Use either a username/password, personal token entered into the username field, or anonymous/public access | |||
// Note: Public access only allows very basic access | |||
func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GitlabDownloader, error) { | |||
gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL)) | |||
gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(&http.Client{ | |||
Transport: &http.Transport{ | |||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}, | |||
Proxy: proxy.Proxy(), | |||
}, | |||
})) | |||
// Only use basic auth if token is blank and password is NOT | |||
// Basic auth will fail with empty strings, but empty token will allow anonymous public API usage | |||
if token == "" && password != "" { | |||
@@ -295,6 +303,13 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea | |||
PublisherName: rel.Author.Username, | |||
} | |||
httpClient := &http.Client{ | |||
Transport: &http.Transport{ | |||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}, | |||
Proxy: proxy.Proxy(), | |||
}, | |||
} | |||
for k, asset := range rel.Assets.Links { | |||
r.Assets = append(r.Assets, &base.ReleaseAsset{ | |||
ID: int64(asset.ID), | |||
@@ -313,8 +328,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea | |||
return nil, err | |||
} | |||
req = req.WithContext(g.ctx) | |||
resp, err := http.DefaultClient.Do(req) | |||
resp, err := httpClient.Do(req) | |||
if err != nil { | |||
return nil, err | |||
} |
@@ -6,6 +6,7 @@ package migrations | |||
import ( | |||
"context" | |||
"crypto/tls" | |||
"fmt" | |||
"net/http" | |||
"net/url" | |||
@@ -14,6 +15,8 @@ import ( | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/migrations/base" | |||
"code.gitea.io/gitea/modules/proxy" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/modules/structs" | |||
"github.com/gogs/go-gogs-client" | |||
@@ -95,9 +98,10 @@ func NewGogsDownloader(ctx context.Context, baseURL, userName, password, token, | |||
downloader.userName = token | |||
} else { | |||
downloader.transport = &http.Transport{ | |||
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}, | |||
Proxy: func(req *http.Request) (*url.URL, error) { | |||
req.SetBasicAuth(userName, password) | |||
return nil, nil | |||
return proxy.Proxy()(req) | |||
}, | |||
} | |||
@@ -0,0 +1,83 @@ | |||
// Copyright 2021 The Gitea Authors. All rights reserved. | |||
// Use of this source code is governed by a MIT-style | |||
// license that can be found in the LICENSE file. | |||
package proxy | |||
import ( | |||
"net/http" | |||
"net/url" | |||
"os" | |||
"sync" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/setting" | |||
"github.com/gobwas/glob" | |||
) | |||
var ( | |||
once sync.Once | |||
hostMatchers []glob.Glob | |||
) | |||
// GetProxyURL returns proxy url | |||
func GetProxyURL() string { | |||
if !setting.Proxy.Enabled { | |||
return "" | |||
} | |||
if setting.Proxy.ProxyURL == "" { | |||
if os.Getenv("http_proxy") != "" { | |||
return os.Getenv("http_proxy") | |||
} | |||
return os.Getenv("https_proxy") | |||
} | |||
return setting.Proxy.ProxyURL | |||
} | |||
// Match return true if url needs to be proxied | |||
func Match(u string) bool { | |||
if !setting.Proxy.Enabled { | |||
return false | |||
} | |||
// enforce do once | |||
Proxy() | |||
for _, v := range hostMatchers { | |||
if v.Match(u) { | |||
return true | |||
} | |||
} | |||
return false | |||
} | |||
// Proxy returns the system proxy | |||
func Proxy() func(req *http.Request) (*url.URL, error) { | |||
if !setting.Proxy.Enabled { | |||
return nil | |||
} | |||
if setting.Proxy.ProxyURL == "" { | |||
return http.ProxyFromEnvironment | |||
} | |||
once.Do(func() { | |||
for _, h := range setting.Proxy.ProxyHosts { | |||
if g, err := glob.Compile(h); err == nil { | |||
hostMatchers = append(hostMatchers, g) | |||
} else { | |||
log.Error("glob.Compile %s failed: %v", h, err) | |||
} | |||
} | |||
}) | |||
return func(req *http.Request) (*url.URL, error) { | |||
for _, v := range hostMatchers { | |||
if v.Match(req.URL.Host) { | |||
return http.ProxyURL(setting.Proxy.ProxyURLFixed)(req) | |||
} | |||
} | |||
return http.ProxyFromEnvironment(req) | |||
} | |||
} |
@@ -126,7 +126,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models. | |||
if opts.LFS { | |||
ep := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint) | |||
if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, ep); err != nil { | |||
if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, ep, setting.Migrations.SkipTLSVerify); err != nil { | |||
log.Error("Failed to store missing LFS objects for repository: %v", err) | |||
} | |||
} | |||
@@ -316,8 +316,8 @@ func PushUpdateAddTag(repo *models.Repository, gitRepo *git.Repository, tagName | |||
} | |||
// StoreMissingLfsObjectsInRepository downloads missing LFS objects | |||
func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, endpoint *url.URL) error { | |||
client := lfs.NewClient(endpoint) | |||
func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, endpoint *url.URL, skipTLSVerify bool) error { | |||
client := lfs.NewClient(endpoint, skipTLSVerify) | |||
contentStore := lfs.NewContentStore() | |||
pointerChan := make(chan lfs.PointerBlob) |
@@ -16,6 +16,7 @@ var ( | |||
AllowedDomains []string | |||
BlockedDomains []string | |||
AllowLocalNetworks bool | |||
SkipTLSVerify bool | |||
}{ | |||
MaxAttempts: 3, | |||
RetryBackoff: 3, | |||
@@ -37,4 +38,5 @@ func newMigrationsService() { | |||
} | |||
Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false) | |||
Migrations.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool(false) | |||
} |
@@ -0,0 +1,40 @@ | |||
// Copyright 2021 The Gitea Authors. All rights reserved. | |||
// Use of this source code is governed by a MIT-style | |||
// license that can be found in the LICENSE file. | |||
package setting | |||
import ( | |||
"net/url" | |||
"code.gitea.io/gitea/modules/log" | |||
) | |||
var ( | |||
// Proxy settings | |||
Proxy = struct { | |||
Enabled bool | |||
ProxyURL string | |||
ProxyURLFixed *url.URL | |||
ProxyHosts []string | |||
}{ | |||
Enabled: false, | |||
ProxyURL: "", | |||
ProxyHosts: []string{}, | |||
} | |||
) | |||
func newProxyService() { | |||
sec := Cfg.Section("proxy") | |||
Proxy.Enabled = sec.Key("PROXY_ENABLED").MustBool(false) | |||
Proxy.ProxyURL = sec.Key("PROXY_URL").MustString("") | |||
if Proxy.ProxyURL != "" { | |||
var err error | |||
Proxy.ProxyURLFixed, err = url.Parse(Proxy.ProxyURL) | |||
if err != nil { | |||
log.Error("Global PROXY_URL is not valid") | |||
Proxy.ProxyURL = "" | |||
} | |||
} | |||
Proxy.ProxyHosts = sec.Key("PROXY_HOSTS").Strings(",") | |||
} |
@@ -1195,6 +1195,7 @@ func NewServices() { | |||
newMailService() | |||
newRegisterMailService() | |||
newNotifyMailService() | |||
newProxyService() | |||
newWebhookService() | |||
newMigrationsService() | |||
newIndexerService() |
@@ -196,7 +196,7 @@ func runSync(ctx context.Context, m *models.Mirror) ([]*mirrorSyncResult, bool) | |||
if m.LFS && setting.LFS.StartServer { | |||
log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo) | |||
ep := lfs.DetermineEndpoint(remoteAddr.String(), m.LFSEndpoint) | |||
if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, ep); err != nil { | |||
if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, ep, false); err != nil { | |||
log.Error("Failed to synchronize LFS objects for repository: %v", err) | |||
} | |||
} |
@@ -134,7 +134,7 @@ func runPushSync(ctx context.Context, m *models.PushMirror) error { | |||
defer gitRepo.Close() | |||
ep := lfs.DetermineEndpoint(remoteAddr.String(), "") | |||
if err := pushAllLFSObjects(ctx, gitRepo, ep); err != nil { | |||
if err := pushAllLFSObjects(ctx, gitRepo, ep, false); err != nil { | |||
return util.NewURLSanitizedError(err, remoteAddr, true) | |||
} | |||
} | |||
@@ -176,8 +176,8 @@ func runPushSync(ctx context.Context, m *models.PushMirror) error { | |||
return nil | |||
} | |||
func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *url.URL) error { | |||
client := lfs.NewClient(endpoint) | |||
func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *url.URL, skipTLSVerify bool) error { | |||
client := lfs.NewClient(endpoint, skipTLSVerify) | |||
contentStore := lfs.NewContentStore() | |||
pointerChan := make(chan lfs.PointerBlob) |
@@ -25,6 +25,7 @@ import ( | |||
"code.gitea.io/gitea/models" | |||
"code.gitea.io/gitea/modules/graceful" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/proxy" | |||
"code.gitea.io/gitea/modules/setting" | |||
"github.com/gobwas/glob" | |||
) | |||
@@ -260,7 +261,7 @@ var ( | |||
func webhookProxy() func(req *http.Request) (*url.URL, error) { | |||
if setting.Webhook.ProxyURL == "" { | |||
return http.ProxyFromEnvironment | |||
return proxy.Proxy() | |||
} | |||
once.Do(func() { |