diff options
Diffstat (limited to 'modules/httplib')
-rw-r--r-- | modules/httplib/request.go | 26 | ||||
-rw-r--r-- | modules/httplib/serve.go | 31 | ||||
-rw-r--r-- | modules/httplib/serve_test.go | 14 | ||||
-rw-r--r-- | modules/httplib/url.go | 106 | ||||
-rw-r--r-- | modules/httplib/url_test.go | 64 |
5 files changed, 185 insertions, 56 deletions
diff --git a/modules/httplib/request.go b/modules/httplib/request.go index 880d7ad3cb..49ea6f4b73 100644 --- a/modules/httplib/request.go +++ b/modules/httplib/request.go @@ -8,6 +8,7 @@ import ( "bytes" "context" "crypto/tls" + "errors" "fmt" "io" "net" @@ -99,18 +100,27 @@ func (r *Request) Param(key, value string) *Request { return r } -// Body adds request raw body. -// it supports string and []byte. +// Body adds request raw body. It supports string, []byte and io.Reader as body. func (r *Request) Body(data any) *Request { + if r == nil { + return nil + } switch t := data.(type) { + case nil: // do nothing case string: - bf := bytes.NewBufferString(t) + bf := strings.NewReader(t) r.req.Body = io.NopCloser(bf) r.req.ContentLength = int64(len(t)) case []byte: bf := bytes.NewBuffer(t) r.req.Body = io.NopCloser(bf) r.req.ContentLength = int64(len(t)) + case io.ReadCloser: + r.req.Body = t + case io.Reader: + r.req.Body = io.NopCloser(t) + default: + panic(fmt.Sprintf("unsupported request body type %T", t)) } return r } @@ -133,15 +143,15 @@ func (r *Request) getResponse() (*http.Response, error) { paramBody = paramBody[0 : len(paramBody)-1] } - if r.req.Method == "GET" && len(paramBody) > 0 { + if r.req.Method == http.MethodGet && len(paramBody) > 0 { if strings.Contains(r.url, "?") { r.url += "&" + paramBody } else { r.url = r.url + "?" + paramBody } - } else if r.req.Method == "POST" && r.req.Body == nil && len(paramBody) > 0 { + } else if r.req.Method == http.MethodPost && r.req.Body == nil && len(paramBody) > 0 { r.Header("Content-Type", "application/x-www-form-urlencoded") - r.Body(paramBody) + r.Body(paramBody) // string } var err error @@ -185,7 +195,11 @@ func (r *Request) getResponse() (*http.Response, error) { } // Response executes request client gets response manually. +// Caller MUST close the response body if no error occurs func (r *Request) Response() (*http.Response, error) { + if r == nil { + return nil, errors.New("invalid request") + } return r.getResponse() } diff --git a/modules/httplib/serve.go b/modules/httplib/serve.go index 8fb667876e..7c1edf432d 100644 --- a/modules/httplib/serve.go +++ b/modules/httplib/serve.go @@ -33,6 +33,7 @@ type ServeHeaderOptions struct { ContentLength *int64 Disposition string // defaults to "attachment" Filename string + CacheIsPublic bool CacheDuration time.Duration // defaults to 5 minutes LastModified time.Time } @@ -72,11 +73,11 @@ func ServeSetHeaders(w http.ResponseWriter, opts *ServeHeaderOptions) { header.Set("Access-Control-Expose-Headers", "Content-Disposition") } - duration := opts.CacheDuration - if duration == 0 { - duration = 5 * time.Minute - } - httpcache.SetCacheControlInHeader(header, duration) + httpcache.SetCacheControlInHeader(header, &httpcache.CacheControlOptions{ + IsPublic: opts.CacheIsPublic, + MaxAge: opts.CacheDuration, + NoTransform: true, + }) if !opts.LastModified.IsZero() { // http.TimeFormat required a UTC time, refer to https://pkg.go.dev/net/http#TimeFormat @@ -85,19 +86,15 @@ func ServeSetHeaders(w http.ResponseWriter, opts *ServeHeaderOptions) { } // ServeData download file from io.Reader -func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, filePath string, mineBuf []byte) { +func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, mineBuf []byte, opts *ServeHeaderOptions) { // do not set "Content-Length", because the length could only be set by callers, and it needs to support range requests - opts := &ServeHeaderOptions{ - Filename: path.Base(filePath), - } - sniffedType := typesniffer.DetectContentType(mineBuf) // the "render" parameter came from year 2016: 638dd24c, it doesn't have clear meaning, so I think it could be removed later isPlain := sniffedType.IsText() || r.FormValue("render") != "" if setting.MimeTypeMap.Enabled { - fileExtension := strings.ToLower(filepath.Ext(filePath)) + fileExtension := strings.ToLower(filepath.Ext(opts.Filename)) opts.ContentType = setting.MimeTypeMap.Map[fileExtension] } @@ -114,7 +111,7 @@ func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, filePath stri if isPlain { charset, err := charsetModule.DetectEncoding(mineBuf) if err != nil { - log.Error("Detect raw file %s charset failed: %v, using by default utf-8", filePath, err) + log.Error("Detect raw file %s charset failed: %v, using by default utf-8", opts.Filename, err) charset = "utf-8" } opts.ContentTypeCharset = strings.ToLower(charset) @@ -142,7 +139,7 @@ func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, filePath stri const mimeDetectionBufferLen = 1024 -func ServeContentByReader(r *http.Request, w http.ResponseWriter, filePath string, size int64, reader io.Reader) { +func ServeContentByReader(r *http.Request, w http.ResponseWriter, size int64, reader io.Reader, opts *ServeHeaderOptions) { buf := make([]byte, mimeDetectionBufferLen) n, err := util.ReadAtMost(reader, buf) if err != nil { @@ -152,7 +149,7 @@ func ServeContentByReader(r *http.Request, w http.ResponseWriter, filePath strin if n >= 0 { buf = buf[:n] } - setServeHeadersByFile(r, w, filePath, buf) + setServeHeadersByFile(r, w, buf, opts) // reset the reader to the beginning reader = io.MultiReader(bytes.NewReader(buf), reader) @@ -215,7 +212,7 @@ func ServeContentByReader(r *http.Request, w http.ResponseWriter, filePath strin _, _ = io.CopyN(w, reader, partialLength) // just like http.ServeContent, not necessary to handle the error } -func ServeContentByReadSeeker(r *http.Request, w http.ResponseWriter, filePath string, modTime *time.Time, reader io.ReadSeeker) { +func ServeContentByReadSeeker(r *http.Request, w http.ResponseWriter, modTime *time.Time, reader io.ReadSeeker, opts *ServeHeaderOptions) { buf := make([]byte, mimeDetectionBufferLen) n, err := util.ReadAtMost(reader, buf) if err != nil { @@ -229,9 +226,9 @@ func ServeContentByReadSeeker(r *http.Request, w http.ResponseWriter, filePath s if n >= 0 { buf = buf[:n] } - setServeHeadersByFile(r, w, filePath, buf) + setServeHeadersByFile(r, w, buf, opts) if modTime == nil { modTime = &time.Time{} } - http.ServeContent(w, r, path.Base(filePath), *modTime, reader) + http.ServeContent(w, r, opts.Filename, *modTime, reader) } diff --git a/modules/httplib/serve_test.go b/modules/httplib/serve_test.go index e53f38b697..78b88c9b5f 100644 --- a/modules/httplib/serve_test.go +++ b/modules/httplib/serve_test.go @@ -4,11 +4,11 @@ package httplib import ( - "fmt" "net/http" "net/http/httptest" "net/url" "os" + "strconv" "strings" "testing" @@ -23,14 +23,14 @@ func TestServeContentByReader(t *testing.T) { _, rangeStr, _ := strings.Cut(t.Name(), "_range_") r := &http.Request{Header: http.Header{}, Form: url.Values{}} if rangeStr != "" { - r.Header.Set("Range", fmt.Sprintf("bytes=%s", rangeStr)) + r.Header.Set("Range", "bytes="+rangeStr) } reader := strings.NewReader(data) w := httptest.NewRecorder() - ServeContentByReader(r, w, "test", int64(len(data)), reader) + ServeContentByReader(r, w, int64(len(data)), reader, &ServeHeaderOptions{}) assert.Equal(t, expectedStatusCode, w.Code) if expectedStatusCode == http.StatusPartialContent || expectedStatusCode == http.StatusOK { - assert.Equal(t, fmt.Sprint(len(expectedContent)), w.Header().Get("Content-Length")) + assert.Equal(t, strconv.Itoa(len(expectedContent)), w.Header().Get("Content-Length")) assert.Equal(t, expectedContent, w.Body.String()) } } @@ -68,7 +68,7 @@ func TestServeContentByReadSeeker(t *testing.T) { _, rangeStr, _ := strings.Cut(t.Name(), "_range_") r := &http.Request{Header: http.Header{}, Form: url.Values{}} if rangeStr != "" { - r.Header.Set("Range", fmt.Sprintf("bytes=%s", rangeStr)) + r.Header.Set("Range", "bytes="+rangeStr) } seekReader, err := os.OpenFile(tmpFile, os.O_RDONLY, 0o644) @@ -76,10 +76,10 @@ func TestServeContentByReadSeeker(t *testing.T) { defer seekReader.Close() w := httptest.NewRecorder() - ServeContentByReadSeeker(r, w, "test", nil, seekReader) + ServeContentByReadSeeker(r, w, nil, seekReader, &ServeHeaderOptions{}) assert.Equal(t, expectedStatusCode, w.Code) if expectedStatusCode == http.StatusPartialContent || expectedStatusCode == http.StatusOK { - assert.Equal(t, fmt.Sprint(len(expectedContent)), w.Header().Get("Content-Length")) + assert.Equal(t, strconv.Itoa(len(expectedContent)), w.Header().Get("Content-Length")) assert.Equal(t, expectedContent, w.Body.String()) } } diff --git a/modules/httplib/url.go b/modules/httplib/url.go index f543c09190..f51506ac3b 100644 --- a/modules/httplib/url.go +++ b/modules/httplib/url.go @@ -53,28 +53,34 @@ func getRequestScheme(req *http.Request) string { return "" } -// GuessCurrentAppURL tries to guess the current full app URL (with sub-path) by http headers. It always has a '/' suffix, exactly the same as setting.AppURL +// GuessCurrentAppURL tries to guess the current full public URL (with sub-path) by http headers. It always has a '/' suffix, exactly the same as setting.AppURL +// TODO: should rename it to GuessCurrentPublicURL in the future func GuessCurrentAppURL(ctx context.Context) string { return GuessCurrentHostURL(ctx) + setting.AppSubURL + "/" } // GuessCurrentHostURL tries to guess the current full host URL (no sub-path) by http headers, there is no trailing slash. func GuessCurrentHostURL(ctx context.Context) string { - req, ok := ctx.Value(RequestContextKey).(*http.Request) - if !ok { - return strings.TrimSuffix(setting.AppURL, setting.AppSubURL+"/") - } - // If no scheme provided by reverse proxy, then do not guess the AppURL, use the configured one. + // Try the best guess to get the current host URL (will be used for public URL) by http headers. // At the moment, if site admin doesn't configure the proxy headers correctly, then Gitea would guess wrong. // There are some cases: // 1. The reverse proxy is configured correctly, it passes "X-Forwarded-Proto/Host" headers. Perfect, Gitea can handle it correctly. // 2. The reverse proxy is not configured correctly, doesn't pass "X-Forwarded-Proto/Host" headers, eg: only one "proxy_pass http://gitea:3000" in Nginx. // 3. There is no reverse proxy. - // Without an extra config option, Gitea is impossible to distinguish between case 2 and case 3, - // then case 2 would result in wrong guess like guessed AppURL becomes "http://gitea:3000/", which is not accessible by end users. - // So in the future maybe it should introduce a new config option, to let site admin decide how to guess the AppURL. + // Without more information, Gitea is impossible to distinguish between case 2 and case 3, then case 2 would result in + // wrong guess like guessed public URL becomes "http://gitea:3000/" behind a "https" reverse proxy, which is not accessible by end users. + // So we introduced "PUBLIC_URL_DETECTION" option, to control the guessing behavior to satisfy different use cases. + req, ok := ctx.Value(RequestContextKey).(*http.Request) + if !ok { + return strings.TrimSuffix(setting.AppURL, setting.AppSubURL+"/") + } reqScheme := getRequestScheme(req) if reqScheme == "" { + // if no reverse proxy header, try to use "Host" header for absolute URL + if setting.PublicURLDetection == setting.PublicURLAuto && req.Host != "" { + return util.Iif(req.TLS == nil, "http://", "https://") + req.Host + } + // fall back to default AppURL return strings.TrimSuffix(setting.AppURL, setting.AppSubURL+"/") } // X-Forwarded-Host has many problems: non-standard, not well-defined (X-Forwarded-Port or not), conflicts with Host header. @@ -88,8 +94,8 @@ func GuessCurrentHostDomain(ctx context.Context) string { return util.IfZero(domain, host) } -// MakeAbsoluteURL tries to make a link to an absolute URL: -// * If link is empty, it returns the current app URL. +// MakeAbsoluteURL tries to make a link to an absolute public URL: +// * If link is empty, it returns the current public URL. // * If link is absolute, it returns the link. // * Otherwise, it returns the current host URL + link, the link itself should have correct sub-path (AppSubURL) if needed. func MakeAbsoluteURL(ctx context.Context, link string) string { @@ -102,25 +108,77 @@ func MakeAbsoluteURL(ctx context.Context, link string) string { return GuessCurrentHostURL(ctx) + "/" + strings.TrimPrefix(link, "/") } -func IsCurrentGiteaSiteURL(ctx context.Context, s string) bool { +type urlType int + +const ( + urlTypeGiteaAbsolute urlType = iota + 1 // "http://gitea/subpath" + urlTypeGiteaPageRelative // "/subpath" + urlTypeGiteaSiteRelative // "?key=val" + urlTypeUnknown // "http://other" +) + +func detectURLRoutePath(ctx context.Context, s string) (routePath string, ut urlType) { u, err := url.Parse(s) if err != nil { - return false + return "", urlTypeUnknown } + cleanedPath := "" if u.Path != "" { - cleanedPath := util.PathJoinRelX(u.Path) - if cleanedPath == "" || cleanedPath == "." { - u.Path = "/" - } else { - u.Path = "/" + cleanedPath + "/" - } + cleanedPath = util.PathJoinRelX(u.Path) + cleanedPath = util.Iif(cleanedPath == ".", "", "/"+cleanedPath) } if urlIsRelative(s, u) { - return u.Path == "" || strings.HasPrefix(strings.ToLower(u.Path), strings.ToLower(setting.AppSubURL+"/")) - } - if u.Path == "" { - u.Path = "/" + if u.Path == "" { + return "", urlTypeGiteaPageRelative + } + if strings.HasPrefix(strings.ToLower(cleanedPath+"/"), strings.ToLower(setting.AppSubURL+"/")) { + return cleanedPath[len(setting.AppSubURL):], urlTypeGiteaSiteRelative + } + return "", urlTypeUnknown } + u.Path = cleanedPath + "/" urlLower := strings.ToLower(u.String()) - return strings.HasPrefix(urlLower, strings.ToLower(setting.AppURL)) || strings.HasPrefix(urlLower, strings.ToLower(GuessCurrentAppURL(ctx))) + if strings.HasPrefix(urlLower, strings.ToLower(setting.AppURL)) { + return cleanedPath[len(setting.AppSubURL):], urlTypeGiteaAbsolute + } + guessedCurURL := GuessCurrentAppURL(ctx) + if strings.HasPrefix(urlLower, strings.ToLower(guessedCurURL)) { + return cleanedPath[len(setting.AppSubURL):], urlTypeGiteaAbsolute + } + return "", urlTypeUnknown +} + +func IsCurrentGiteaSiteURL(ctx context.Context, s string) bool { + _, ut := detectURLRoutePath(ctx, s) + return ut != urlTypeUnknown +} + +type GiteaSiteURL struct { + RoutePath string + OwnerName string + RepoName string + RepoSubPath string +} + +func ParseGiteaSiteURL(ctx context.Context, s string) *GiteaSiteURL { + routePath, ut := detectURLRoutePath(ctx, s) + if ut == urlTypeUnknown || ut == urlTypeGiteaPageRelative { + return nil + } + ret := &GiteaSiteURL{RoutePath: routePath} + fields := strings.SplitN(strings.TrimPrefix(ret.RoutePath, "/"), "/", 3) + + // TODO: now it only does a quick check for some known reserved paths, should do more strict checks in the future + if fields[0] == "attachments" { + return ret + } + if len(fields) < 2 { + return ret + } + ret.OwnerName = fields[0] + ret.RepoName = fields[1] + if len(fields) == 3 { + ret.RepoSubPath = "/" + fields[2] + } + return ret } diff --git a/modules/httplib/url_test.go b/modules/httplib/url_test.go index fc6c91cd3a..0ffb0cac05 100644 --- a/modules/httplib/url_test.go +++ b/modules/httplib/url_test.go @@ -5,6 +5,7 @@ package httplib import ( "context" + "crypto/tls" "net/http" "testing" @@ -39,12 +40,48 @@ func TestIsRelativeURL(t *testing.T) { } } +func TestGuessCurrentHostURL(t *testing.T) { + defer test.MockVariableValue(&setting.AppURL, "http://cfg-host/sub/")() + defer test.MockVariableValue(&setting.AppSubURL, "/sub")() + headersWithProto := http.Header{"X-Forwarded-Proto": {"https"}} + + t.Run("Legacy", func(t *testing.T) { + defer test.MockVariableValue(&setting.PublicURLDetection, setting.PublicURLLegacy)() + + assert.Equal(t, "http://cfg-host", GuessCurrentHostURL(t.Context())) + + // legacy: "Host" is not used when there is no "X-Forwarded-Proto" header + ctx := context.WithValue(t.Context(), RequestContextKey, &http.Request{Host: "req-host:3000"}) + assert.Equal(t, "http://cfg-host", GuessCurrentHostURL(ctx)) + + // if "X-Forwarded-Proto" exists, then use it and "Host" header + ctx = context.WithValue(t.Context(), RequestContextKey, &http.Request{Host: "req-host:3000", Header: headersWithProto}) + assert.Equal(t, "https://req-host:3000", GuessCurrentHostURL(ctx)) + }) + + t.Run("Auto", func(t *testing.T) { + defer test.MockVariableValue(&setting.PublicURLDetection, setting.PublicURLAuto)() + + assert.Equal(t, "http://cfg-host", GuessCurrentHostURL(t.Context())) + + // auto: always use "Host" header, the scheme is determined by "X-Forwarded-Proto" header, or TLS config if no "X-Forwarded-Proto" header + ctx := context.WithValue(t.Context(), RequestContextKey, &http.Request{Host: "req-host:3000"}) + assert.Equal(t, "http://req-host:3000", GuessCurrentHostURL(ctx)) + + ctx = context.WithValue(t.Context(), RequestContextKey, &http.Request{Host: "req-host", TLS: &tls.ConnectionState{}}) + assert.Equal(t, "https://req-host", GuessCurrentHostURL(ctx)) + + ctx = context.WithValue(t.Context(), RequestContextKey, &http.Request{Host: "req-host:3000", Header: headersWithProto}) + assert.Equal(t, "https://req-host:3000", GuessCurrentHostURL(ctx)) + }) +} + func TestMakeAbsoluteURL(t *testing.T) { defer test.MockVariableValue(&setting.Protocol, "http")() defer test.MockVariableValue(&setting.AppURL, "http://cfg-host/sub/")() defer test.MockVariableValue(&setting.AppSubURL, "/sub")() - ctx := context.Background() + ctx := t.Context() assert.Equal(t, "http://cfg-host/sub/", MakeAbsoluteURL(ctx, "")) assert.Equal(t, "http://cfg-host/foo", MakeAbsoluteURL(ctx, "foo")) assert.Equal(t, "http://cfg-host/foo", MakeAbsoluteURL(ctx, "/foo")) @@ -76,7 +113,7 @@ func TestMakeAbsoluteURL(t *testing.T) { func TestIsCurrentGiteaSiteURL(t *testing.T) { defer test.MockVariableValue(&setting.AppURL, "http://localhost:3000/sub/")() defer test.MockVariableValue(&setting.AppSubURL, "/sub")() - ctx := context.Background() + ctx := t.Context() good := []string{ "?key=val", "/sub", @@ -122,3 +159,26 @@ func TestIsCurrentGiteaSiteURL(t *testing.T) { assert.True(t, IsCurrentGiteaSiteURL(ctx, "https://user-host")) assert.False(t, IsCurrentGiteaSiteURL(ctx, "https://forwarded-host")) } + +func TestParseGiteaSiteURL(t *testing.T) { + defer test.MockVariableValue(&setting.AppURL, "http://localhost:3000/sub/")() + defer test.MockVariableValue(&setting.AppSubURL, "/sub")() + ctx := t.Context() + tests := []struct { + url string + exp *GiteaSiteURL + }{ + {"http://localhost:3000/sub?k=v", &GiteaSiteURL{RoutePath: ""}}, + {"http://localhost:3000/sub/", &GiteaSiteURL{RoutePath: ""}}, + {"http://localhost:3000/sub/foo", &GiteaSiteURL{RoutePath: "/foo"}}, + {"http://localhost:3000/sub/foo/bar", &GiteaSiteURL{RoutePath: "/foo/bar", OwnerName: "foo", RepoName: "bar"}}, + {"http://localhost:3000/sub/foo/bar/", &GiteaSiteURL{RoutePath: "/foo/bar", OwnerName: "foo", RepoName: "bar"}}, + {"http://localhost:3000/sub/attachments/bar", &GiteaSiteURL{RoutePath: "/attachments/bar"}}, + {"http://localhost:3000/other", nil}, + {"http://other/", nil}, + } + for _, test := range tests { + su := ParseGiteaSiteURL(ctx, test.url) + assert.Equal(t, test.exp, su, "URL = %s", test.url) + } +} |