Add proxy settings and support for migration and webhook (#16704)

* Add proxy settings and support for migration and webhook

* Fix default value

* Add newline for example ini

* Add lfs proxy support

* Fix lint

* Follow @zeripath's review

* Fix git clone

* Fix test

* missgin http requests for proxy

* use empty

Co-authored-by: zeripath <art27@cantab.net>

Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: zeripath <art27@cantab.net>

14 files changed, 258 insertions, 34 deletions

diff --git a/modules/git/command.go b/modules/git/command.go
index d83c42fdc2..e7496f072c 100644
--- a/modules/git/command.go
+++ b/modules/git/command.go
@@ -110,24 +110,47 @@ func (c *Command) RunInDirTimeoutEnvFullPipeline(env []string, timeout time.Dura
 // RunInDirTimeoutEnvFullPipelineFunc executes the command in given directory with given timeout,
 // it pipes stdout and stderr to given io.Writer and passes in an io.Reader as stdin. Between cmd.Start and cmd.Wait the passed in function is run.
 func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time.Duration, dir string, stdout, stderr io.Writer, stdin io.Reader, fn func(context.Context, context.CancelFunc) error) error {
-	if timeout == -1 {
-		timeout = defaultCommandExecutionTimeout
+	return c.RunWithContext(&RunContext{
+		Env:          env,
+		Timeout:      timeout,
+		Dir:          dir,
+		Stdout:       stdout,
+		Stderr:       stderr,
+		Stdin:        stdin,
+		PipelineFunc: fn,
+	})
+}
+
+// RunContext represents parameters to run the command
+type RunContext struct {
+	Env            []string
+	Timeout        time.Duration
+	Dir            string
+	Stdout, Stderr io.Writer
+	Stdin          io.Reader
+	PipelineFunc   func(context.Context, context.CancelFunc) error
+}
+
+// RunWithContext run the command with context
+func (c *Command) RunWithContext(rc *RunContext) error {
+	if rc.Timeout == -1 {
+		rc.Timeout = defaultCommandExecutionTimeout
 	}
 
-	if len(dir) == 0 {
+	if len(rc.Dir) == 0 {
 		log.Debug("%s", c)
 	} else {
-		log.Debug("%s: %v", dir, c)
+		log.Debug("%s: %v", rc.Dir, c)
 	}
 
-	ctx, cancel := context.WithTimeout(c.parentContext, timeout)
+	ctx, cancel := context.WithTimeout(c.parentContext, rc.Timeout)
 	defer cancel()
 
 	cmd := exec.CommandContext(ctx, c.name, c.args...)
-	if env == nil {
+	if rc.Env == nil {
 		cmd.Env = os.Environ()
 	} else {
-		cmd.Env = env
+		cmd.Env = rc.Env
 	}
 
 	cmd.Env = append(
@@ -141,23 +164,23 @@ func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time.
 	if goVersionLessThan115 {
 		cmd.Env = append(cmd.Env, "GODEBUG=asyncpreemptoff=1")
 	}
-	cmd.Dir = dir
-	cmd.Stdout = stdout
-	cmd.Stderr = stderr
-	cmd.Stdin = stdin
+	cmd.Dir = rc.Dir
+	cmd.Stdout = rc.Stdout
+	cmd.Stderr = rc.Stderr
+	cmd.Stdin = rc.Stdin
 	if err := cmd.Start(); err != nil {
 		return err
 	}
 
 	desc := c.desc
 	if desc == "" {
-		desc = fmt.Sprintf("%s %s %s [repo_path: %s]", GitExecutable, c.name, strings.Join(c.args, " "), dir)
+		desc = fmt.Sprintf("%s %s %s [repo_path: %s]", GitExecutable, c.name, strings.Join(c.args, " "), rc.Dir)
 	}
 	pid := process.GetManager().Add(desc, cancel)
 	defer process.GetManager().Remove(pid)
 
-	if fn != nil {
-		err := fn(ctx, cancel)
+	if rc.PipelineFunc != nil {
+		err := rc.PipelineFunc(ctx, cancel)
 		if err != nil {
 			cancel()
 			_ = cmd.Wait()
diff --git a/modules/git/repo.go b/modules/git/repo.go
index 4e6f90c3ef..f2bbbf4716 100644
--- a/modules/git/repo.go
+++ b/modules/git/repo.go
@@ -9,11 +9,15 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
+	"net/url"
 	"os"
 	"path"
 	"strconv"
 	"strings"
 	"time"
+
+	"code.gitea.io/gitea/modules/proxy"
 )
 
 // GPGSettings represents the default GPG settings for this repository
@@ -99,12 +103,12 @@ type CloneRepoOptions struct {
 }
 
 // Clone clones original repository to target path.
-func Clone(from, to string, opts CloneRepoOptions) (err error) {
+func Clone(from, to string, opts CloneRepoOptions) error {
 	return CloneWithContext(DefaultContext, from, to, opts)
 }
 
 // CloneWithContext clones original repository to target path.
-func CloneWithContext(ctx context.Context, from, to string, opts CloneRepoOptions) (err error) {
+func CloneWithContext(ctx context.Context, from, to string, opts CloneRepoOptions) error {
 	cargs := make([]string, len(GlobalCommandArgs))
 	copy(cargs, GlobalCommandArgs)
 	return CloneWithArgs(ctx, from, to, cargs, opts)
@@ -146,8 +150,24 @@ func CloneWithArgs(ctx context.Context, from, to string, args []string, opts Clo
 		opts.Timeout = -1
 	}
 
-	_, err = cmd.RunTimeout(opts.Timeout)
-	return err
+	var envs = os.Environ()
+	u, err := url.Parse(from)
+	if err == nil && (strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https")) {
+		if proxy.Match(u.Host) {
+			envs = append(envs, fmt.Sprintf("https_proxy=%s", proxy.GetProxyURL()))
+		}
+	}
+
+	var stderr = new(bytes.Buffer)
+	if err = cmd.RunWithContext(&RunContext{
+		Timeout: opts.Timeout,
+		Env:     envs,
+		Stdout:  io.Discard,
+		Stderr:  stderr,
+	}); err != nil {
+		return ConcatenateError(err, stderr.String())
+	}
+	return nil
 }
 
 // PullRemoteOptions options when pull from remote
diff --git a/modules/lfs/client.go b/modules/lfs/client.go
index 0a21440f73..81b047c5bd 100644
--- a/modules/lfs/client.go
+++ b/modules/lfs/client.go
@@ -24,9 +24,9 @@ type Client interface {
 }
 
 // NewClient creates a LFS client
-func NewClient(endpoint *url.URL) Client {
+func NewClient(endpoint *url.URL, skipTLSVerify bool) Client {
 	if endpoint.Scheme == "file" {
 		return newFilesystemClient(endpoint)
 	}
-	return newHTTPClient(endpoint)
+	return newHTTPClient(endpoint, skipTLSVerify)
 }
diff --git a/modules/lfs/client_test.go b/modules/lfs/client_test.go
index 1040b39925..ee6b7a59fc 100644
--- a/modules/lfs/client_test.go
+++ b/modules/lfs/client_test.go
@@ -13,10 +13,10 @@ import (
 
 func TestNewClient(t *testing.T) {
 	u, _ := url.Parse("file:///test")
-	c := NewClient(u)
+	c := NewClient(u, true)
 	assert.IsType(t, &FilesystemClient{}, c)
 
 	u, _ = url.Parse("https://test.com/lfs")
-	c = NewClient(u)
+	c = NewClient(u, true)
 	assert.IsType(t, &HTTPClient{}, c)
 }
diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go
index 31c67903a8..5df5ed33a9 100644
--- a/modules/lfs/http_client.go
+++ b/modules/lfs/http_client.go
@@ -7,6 +7,7 @@ package lfs
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"net/http"
@@ -15,6 +16,7 @@ import (
 
 	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/proxy"
 )
 
 const batchSize = 20
@@ -32,8 +34,13 @@ func (c *HTTPClient) BatchSize() int {
 	return batchSize
 }
 
-func newHTTPClient(endpoint *url.URL) *HTTPClient {
-	hc := &http.Client{}
+func newHTTPClient(endpoint *url.URL, skipTLSVerify bool) *HTTPClient {
+	hc := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: skipTLSVerify},
+			Proxy:           proxy.Proxy(),
+		},
+	}
 
 	client := &HTTPClient{
 		client:    hc,
diff --git a/modules/migrations/gitea_downloader.go b/modules/migrations/gitea_downloader.go
index 2ed6c9113d..23ede93a42 100644
--- a/modules/migrations/gitea_downloader.go
+++ b/modules/migrations/gitea_downloader.go
@@ -6,6 +6,7 @@ package migrations
 
 import (
 	"context"
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"io"
@@ -17,6 +18,8 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/proxy"
+	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/structs"
 
 	gitea_sdk "code.gitea.io/sdk/gitea"
@@ -87,6 +90,12 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo
 		gitea_sdk.SetToken(token),
 		gitea_sdk.SetBasicAuth(username, password),
 		gitea_sdk.SetContext(ctx),
+		gitea_sdk.SetHTTPClient(&http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
+				Proxy:           proxy.Proxy(),
+			},
+		}),
 	)
 	if err != nil {
 		log.Error(fmt.Sprintf("Failed to create NewGiteaDownloader for: %s. Error: %v", baseURL, err))
@@ -266,6 +275,13 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele
 		Created:         rel.CreatedAt,
 	}
 
+	httpClient := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
+			Proxy:           proxy.Proxy(),
+		},
+	}
+
 	for _, asset := range rel.Attachments {
 		size := int(asset.Size)
 		dlCount := int(asset.DownloadCount)
@@ -282,7 +298,11 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele
 					return nil, err
 				}
 				// FIXME: for a private download?
-				resp, err := http.Get(asset.DownloadURL)
+				req, err := http.NewRequest("GET", asset.DownloadURL, nil)
+				if err != nil {
+					return nil, err
+				}
+				resp, err := httpClient.Do(req)
 				if err != nil {
 					return nil, err
 				}
diff --git a/modules/migrations/github.go b/modules/migrations/github.go
index cc5279e38f..f6063b0661 100644
--- a/modules/migrations/github.go
+++ b/modules/migrations/github.go
@@ -7,6 +7,7 @@ package migrations
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"io"
 	"net/http"
@@ -17,6 +18,8 @@ import (
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/proxy"
+	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/util"
 
@@ -90,7 +93,7 @@ func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, tok
 		Transport: &http.Transport{
 			Proxy: func(req *http.Request) (*url.URL, error) {
 				req.SetBasicAuth(userName, password)
-				return nil, nil
+				return proxy.Proxy()(req)
 			},
 		},
 	}
@@ -269,6 +272,13 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
 		r.Published = rel.PublishedAt.Time
 	}
 
+	httpClient := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
+			Proxy:           proxy.Proxy(),
+		},
+	}
+
 	for _, asset := range rel.Assets {
 		var assetID = *asset.ID // Don't optimize this, for closure we need a local variable
 		r.Assets = append(r.Assets, &base.ReleaseAsset{
@@ -295,7 +305,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
 						if err != nil {
 							return nil, err
 						}
-						resp, err := http.DefaultClient.Do(req)
+						resp, err := httpClient.Do(req)
 						err1 := g.RefreshRate()
 						if err1 != nil {
 							log.Error("g.client.RateLimits: %s", err1)
diff --git a/modules/migrations/gitlab.go b/modules/migrations/gitlab.go
index 1050ffd0c9..28e9eac63c 100644
--- a/modules/migrations/gitlab.go
+++ b/modules/migrations/gitlab.go
@@ -6,6 +6,7 @@ package migrations
 
 import (
 	"context"
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"io"
@@ -17,6 +18,8 @@ import (
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/proxy"
+	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/structs"
 
 	"github.com/xanzy/go-gitlab"
@@ -77,7 +80,12 @@ type GitlabDownloader struct {
 //   Use either a username/password, personal token entered into the username field, or anonymous/public access
 //   Note: Public access only allows very basic access
 func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GitlabDownloader, error) {
-	gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL))
+	gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(&http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
+			Proxy:           proxy.Proxy(),
+		},
+	}))
 	// Only use basic auth if token is blank and password is NOT
 	// Basic auth will fail with empty strings, but empty token will allow anonymous public API usage
 	if token == "" && password != "" {
@@ -295,6 +303,13 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
 		PublisherName:   rel.Author.Username,
 	}
 
+	httpClient := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
+			Proxy:           proxy.Proxy(),
+		},
+	}
+
 	for k, asset := range rel.Assets.Links {
 		r.Assets = append(r.Assets, &base.ReleaseAsset{
 			ID:            int64(asset.ID),
@@ -313,8 +328,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
 					return nil, err
 				}
 				req = req.WithContext(g.ctx)
-
-				resp, err := http.DefaultClient.Do(req)
+				resp, err := httpClient.Do(req)
 				if err != nil {
 					return nil, err
 				}
diff --git a/modules/migrations/gogs.go b/modules/migrations/gogs.go
index 388020c88a..2c7fa76146 100644
--- a/modules/migrations/gogs.go
+++ b/modules/migrations/gogs.go
@@ -6,6 +6,7 @@ package migrations
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -14,6 +15,8 @@ import (
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/proxy"
+	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/structs"
 
 	"github.com/gogs/go-gogs-client"
@@ -95,9 +98,10 @@ func NewGogsDownloader(ctx context.Context, baseURL, userName, password, token,
 		downloader.userName = token
 	} else {
 		downloader.transport = &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
 			Proxy: func(req *http.Request) (*url.URL, error) {
 				req.SetBasicAuth(userName, password)
-				return nil, nil
+				return proxy.Proxy()(req)
 			},
 		}
 
diff --git a/modules/proxy/proxy.go b/modules/proxy/proxy.go
new file mode 100644
index 0000000000..0ab6ed3341
--- /dev/null
+++ b/modules/proxy/proxy.go
@@ -0,0 +1,83 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package proxy
+
+import (
+	"net/http"
+	"net/url"
+	"os"
+	"sync"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/gobwas/glob"
+)
+
+var (
+	once         sync.Once
+	hostMatchers []glob.Glob
+)
+
+// GetProxyURL returns proxy url
+func GetProxyURL() string {
+	if !setting.Proxy.Enabled {
+		return ""
+	}
+
+	if setting.Proxy.ProxyURL == "" {
+		if os.Getenv("http_proxy") != "" {
+			return os.Getenv("http_proxy")
+		}
+		return os.Getenv("https_proxy")
+	}
+	return setting.Proxy.ProxyURL
+}
+
+// Match return true if url needs to be proxied
+func Match(u string) bool {
+	if !setting.Proxy.Enabled {
+		return false
+	}
+
+	// enforce do once
+	Proxy()
+
+	for _, v := range hostMatchers {
+		if v.Match(u) {
+			return true
+		}
+	}
+	return false
+}
+
+// Proxy returns the system proxy
+func Proxy() func(req *http.Request) (*url.URL, error) {
+	if !setting.Proxy.Enabled {
+		return nil
+	}
+	if setting.Proxy.ProxyURL == "" {
+		return http.ProxyFromEnvironment
+	}
+
+	once.Do(func() {
+		for _, h := range setting.Proxy.ProxyHosts {
+			if g, err := glob.Compile(h); err == nil {
+				hostMatchers = append(hostMatchers, g)
+			} else {
+				log.Error("glob.Compile %s failed: %v", h, err)
+			}
+		}
+	})
+
+	return func(req *http.Request) (*url.URL, error) {
+		for _, v := range hostMatchers {
+			if v.Match(req.URL.Host) {
+				return http.ProxyURL(setting.Proxy.ProxyURLFixed)(req)
+			}
+		}
+		return http.ProxyFromEnvironment(req)
+	}
+}
diff --git a/modules/repository/repo.go b/modules/repository/repo.go
index 08531c04ed..6b87039775 100644
--- a/modules/repository/repo.go
+++ b/modules/repository/repo.go
@@ -126,7 +126,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models.
 
 		if opts.LFS {
 			ep := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint)
-			if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, ep); err != nil {
+			if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, ep, setting.Migrations.SkipTLSVerify); err != nil {
 				log.Error("Failed to store missing LFS objects for repository: %v", err)
 			}
 		}
@@ -316,8 +316,8 @@ func PushUpdateAddTag(repo *models.Repository, gitRepo *git.Repository, tagName
 }
 
 // StoreMissingLfsObjectsInRepository downloads missing LFS objects
-func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, endpoint *url.URL) error {
-	client := lfs.NewClient(endpoint)
+func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, endpoint *url.URL, skipTLSVerify bool) error {
+	client := lfs.NewClient(endpoint, skipTLSVerify)
 	contentStore := lfs.NewContentStore()
 
 	pointerChan := make(chan lfs.PointerBlob)
diff --git a/modules/setting/migrations.go b/modules/setting/migrations.go
index 7808df5280..b663b52f89 100644
--- a/modules/setting/migrations.go
+++ b/modules/setting/migrations.go
@@ -16,6 +16,7 @@ var (
 		AllowedDomains     []string
 		BlockedDomains     []string
 		AllowLocalNetworks bool
+		SkipTLSVerify      bool
 	}{
 		MaxAttempts:  3,
 		RetryBackoff: 3,
@@ -37,4 +38,5 @@ func newMigrationsService() {
 	}
 
 	Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false)
+	Migrations.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool(false)
 }
diff --git a/modules/setting/proxy.go b/modules/setting/proxy.go
new file mode 100644
index 0000000000..b99237a398
--- /dev/null
+++ b/modules/setting/proxy.go
@@ -0,0 +1,40 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package setting
+
+import (
+	"net/url"
+
+	"code.gitea.io/gitea/modules/log"
+)
+
+var (
+	// Proxy settings
+	Proxy = struct {
+		Enabled       bool
+		ProxyURL      string
+		ProxyURLFixed *url.URL
+		ProxyHosts    []string
+	}{
+		Enabled:    false,
+		ProxyURL:   "",
+		ProxyHosts: []string{},
+	}
+)
+
+func newProxyService() {
+	sec := Cfg.Section("proxy")
+	Proxy.Enabled = sec.Key("PROXY_ENABLED").MustBool(false)
+	Proxy.ProxyURL = sec.Key("PROXY_URL").MustString("")
+	if Proxy.ProxyURL != "" {
+		var err error
+		Proxy.ProxyURLFixed, err = url.Parse(Proxy.ProxyURL)
+		if err != nil {
+			log.Error("Global PROXY_URL is not valid")
+			Proxy.ProxyURL = ""
+		}
+	}
+	Proxy.ProxyHosts = sec.Key("PROXY_HOSTS").Strings(",")
+}
diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index d584ed3d4d..441bceda20 100644
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -1195,6 +1195,7 @@ func NewServices() {
 	newMailService()
 	newRegisterMailService()
 	newNotifyMailService()
+	newProxyService()
 	newWebhookService()
 	newMigrationsService()
 	newIndexerService()