Make repository response support HTTP range request (#24592)

Replace #20480
Replace #18448

Close #16414

7 files changed, 381 insertions, 79 deletions

diff --git a/modules/context/context_serve.go b/modules/context/context_serve.go
index 44dd739eff..5569efbc7e 100644
--- a/modules/context/context_serve.go
+++ b/modules/context/context_serve.go
@@ -4,71 +4,20 @@
 package context
 
 import (
-	"fmt"
 	"io"
 	"net/http"
-	"net/url"
-	"strconv"
-	"strings"
-	"time"
 
-	"code.gitea.io/gitea/modules/httpcache"
-	"code.gitea.io/gitea/modules/typesniffer"
+	"code.gitea.io/gitea/modules/httplib"
 )
 
-type ServeHeaderOptions struct {
-	ContentType        string // defaults to "application/octet-stream"
-	ContentTypeCharset string
-	ContentLength      *int64
-	Disposition        string // defaults to "attachment"
-	Filename           string
-	CacheDuration      time.Duration // defaults to 5 minutes
-	LastModified       time.Time
-}
-
-// SetServeHeaders sets necessary content serve headers
-func (ctx *Context) SetServeHeaders(opts *ServeHeaderOptions) {
-	header := ctx.Resp.Header()
-
-	contentType := typesniffer.ApplicationOctetStream
-	if opts.ContentType != "" {
-		if opts.ContentTypeCharset != "" {
-			contentType = opts.ContentType + "; charset=" + strings.ToLower(opts.ContentTypeCharset)
-		} else {
-			contentType = opts.ContentType
-		}
-	}
-	header.Set("Content-Type", contentType)
-	header.Set("X-Content-Type-Options", "nosniff")
-
-	if opts.ContentLength != nil {
-		header.Set("Content-Length", strconv.FormatInt(*opts.ContentLength, 10))
-	}
-
-	if opts.Filename != "" {
-		disposition := opts.Disposition
-		if disposition == "" {
-			disposition = "attachment"
-		}
-
-		backslashEscapedName := strings.ReplaceAll(strings.ReplaceAll(opts.Filename, `\`, `\\`), `"`, `\"`) // \ -> \\, " -> \"
-		header.Set("Content-Disposition", fmt.Sprintf(`%s; filename="%s"; filename*=UTF-8''%s`, disposition, backslashEscapedName, url.PathEscape(opts.Filename)))
-		header.Set("Access-Control-Expose-Headers", "Content-Disposition")
-	}
-
-	duration := opts.CacheDuration
-	if duration == 0 {
-		duration = 5 * time.Minute
-	}
-	httpcache.SetCacheControlInHeader(header, duration)
+type ServeHeaderOptions httplib.ServeHeaderOptions
 
-	if !opts.LastModified.IsZero() {
-		header.Set("Last-Modified", opts.LastModified.UTC().Format(http.TimeFormat))
-	}
+func (ctx *Context) SetServeHeaders(opt *ServeHeaderOptions) {
+	httplib.ServeSetHeaders(ctx.Resp, (*httplib.ServeHeaderOptions)(opt))
 }
 
 // ServeContent serves content to http request
 func (ctx *Context) ServeContent(r io.ReadSeeker, opts *ServeHeaderOptions) {
-	ctx.SetServeHeaders(opts)
+	httplib.ServeSetHeaders(ctx.Resp, (*httplib.ServeHeaderOptions)(opts))
 	http.ServeContent(ctx.Resp, ctx.Req, opts.Filename, opts.LastModified, r)
 }
diff --git a/modules/context/context_test.go b/modules/context/context_test.go
index e1460c1fd7..a6facc9788 100644
--- a/modules/context/context_test.go
+++ b/modules/context/context_test.go
@@ -7,32 +7,16 @@ import (
 	"net/http"
 	"testing"
 
+	"code.gitea.io/gitea/modules/httplib"
 	"code.gitea.io/gitea/modules/setting"
 
 	"github.com/stretchr/testify/assert"
 )
 
-type mockResponseWriter struct {
-	header http.Header
-}
-
-func (m *mockResponseWriter) Header() http.Header {
-	return m.header
-}
-
-func (m *mockResponseWriter) Write(bytes []byte) (int, error) {
-	panic("implement me")
-}
-
-func (m *mockResponseWriter) WriteHeader(statusCode int) {
-	panic("implement me")
-}
-
 func TestRemoveSessionCookieHeader(t *testing.T) {
-	w := &mockResponseWriter{}
-	w.header = http.Header{}
-	w.header.Add("Set-Cookie", (&http.Cookie{Name: setting.SessionConfig.CookieName, Value: "foo"}).String())
-	w.header.Add("Set-Cookie", (&http.Cookie{Name: "other", Value: "bar"}).String())
+	w := httplib.NewMockResponseWriter()
+	w.Header().Add("Set-Cookie", (&http.Cookie{Name: setting.SessionConfig.CookieName, Value: "foo"}).String())
+	w.Header().Add("Set-Cookie", (&http.Cookie{Name: "other", Value: "bar"}).String())
 	assert.Len(t, w.Header().Values("Set-Cookie"), 2)
 	removeSessionCookieHeader(w)
 	assert.Len(t, w.Header().Values("Set-Cookie"), 1)
diff --git a/modules/httplib/mock.go b/modules/httplib/mock.go
new file mode 100644
index 0000000000..7d284e86fb
--- /dev/null
+++ b/modules/httplib/mock.go
@@ -0,0 +1,35 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package httplib
+
+import (
+	"bytes"
+	"net/http"
+)
+
+type MockResponseWriter struct {
+	header http.Header
+
+	StatusCode int
+	BodyBuffer bytes.Buffer
+}
+
+func (m *MockResponseWriter) Header() http.Header {
+	return m.header
+}
+
+func (m *MockResponseWriter) Write(bytes []byte) (int, error) {
+	if m.StatusCode == 0 {
+		m.StatusCode = http.StatusOK
+	}
+	return m.BodyBuffer.Write(bytes)
+}
+
+func (m *MockResponseWriter) WriteHeader(statusCode int) {
+	m.StatusCode = statusCode
+}
+
+func NewMockResponseWriter() *MockResponseWriter {
+	return &MockResponseWriter{header: http.Header{}}
+}
diff --git a/modules/httplib/httplib.go b/modules/httplib/request.go
index e904d77e14..e904d77e14 100644
--- a/modules/httplib/httplib.go
+++ b/modules/httplib/request.go
diff --git a/modules/httplib/serve.go b/modules/httplib/serve.go
new file mode 100644
index 0000000000..12d68c2d65
--- /dev/null
+++ b/modules/httplib/serve.go
@@ -0,0 +1,225 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package httplib
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	charsetModule "code.gitea.io/gitea/modules/charset"
+	"code.gitea.io/gitea/modules/httpcache"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/typesniffer"
+	"code.gitea.io/gitea/modules/util"
+)
+
+type ServeHeaderOptions struct {
+	ContentType        string // defaults to "application/octet-stream"
+	ContentTypeCharset string
+	ContentLength      *int64
+	Disposition        string // defaults to "attachment"
+	Filename           string
+	CacheDuration      time.Duration // defaults to 5 minutes
+	LastModified       time.Time
+}
+
+// ServeSetHeaders sets necessary content serve headers
+func ServeSetHeaders(w http.ResponseWriter, opts *ServeHeaderOptions) {
+	header := w.Header()
+
+	contentType := typesniffer.ApplicationOctetStream
+	if opts.ContentType != "" {
+		if opts.ContentTypeCharset != "" {
+			contentType = opts.ContentType + "; charset=" + strings.ToLower(opts.ContentTypeCharset)
+		} else {
+			contentType = opts.ContentType
+		}
+	}
+	header.Set("Content-Type", contentType)
+	header.Set("X-Content-Type-Options", "nosniff")
+
+	if opts.ContentLength != nil {
+		header.Set("Content-Length", strconv.FormatInt(*opts.ContentLength, 10))
+	}
+
+	if opts.Filename != "" {
+		disposition := opts.Disposition
+		if disposition == "" {
+			disposition = "attachment"
+		}
+
+		backslashEscapedName := strings.ReplaceAll(strings.ReplaceAll(opts.Filename, `\`, `\\`), `"`, `\"`) // \ -> \\, " -> \"
+		header.Set("Content-Disposition", fmt.Sprintf(`%s; filename="%s"; filename*=UTF-8''%s`, disposition, backslashEscapedName, url.PathEscape(opts.Filename)))
+		header.Set("Access-Control-Expose-Headers", "Content-Disposition")
+	}
+
+	duration := opts.CacheDuration
+	if duration == 0 {
+		duration = 5 * time.Minute
+	}
+	httpcache.SetCacheControlInHeader(header, duration)
+
+	if !opts.LastModified.IsZero() {
+		header.Set("Last-Modified", opts.LastModified.UTC().Format(http.TimeFormat))
+	}
+}
+
+// ServeData download file from io.Reader
+func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, filePath string, mineBuf []byte) {
+	// do not set "Content-Length", because the length could only be set by callers, and it needs to support range requests
+	opts := &ServeHeaderOptions{
+		Filename: path.Base(filePath),
+	}
+
+	sniffedType := typesniffer.DetectContentType(mineBuf)
+
+	// the "render" parameter came from year 2016: 638dd24c, it doesn't have clear meaning, so I think it could be removed later
+	isPlain := sniffedType.IsText() || r.FormValue("render") != ""
+
+	if setting.MimeTypeMap.Enabled {
+		fileExtension := strings.ToLower(filepath.Ext(filePath))
+		opts.ContentType = setting.MimeTypeMap.Map[fileExtension]
+	}
+
+	if opts.ContentType == "" {
+		if sniffedType.IsBrowsableBinaryType() {
+			opts.ContentType = sniffedType.GetMimeType()
+		} else if isPlain {
+			opts.ContentType = "text/plain"
+		} else {
+			opts.ContentType = typesniffer.ApplicationOctetStream
+		}
+	}
+
+	if isPlain {
+		charset, err := charsetModule.DetectEncoding(mineBuf)
+		if err != nil {
+			log.Error("Detect raw file %s charset failed: %v, using by default utf-8", filePath, err)
+			charset = "utf-8"
+		}
+		opts.ContentTypeCharset = strings.ToLower(charset)
+	}
+
+	isSVG := sniffedType.IsSvgImage()
+
+	// serve types that can present a security risk with CSP
+	if isSVG {
+		w.Header().Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'; sandbox")
+	} else if sniffedType.IsPDF() {
+		// no sandbox attribute for pdf as it breaks rendering in at least safari. this
+		// should generally be safe as scripts inside PDF can not escape the PDF document
+		// see https://bugs.chromium.org/p/chromium/issues/detail?id=413851 for more discussion
+		w.Header().Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'")
+	}
+
+	opts.Disposition = "inline"
+	if isSVG && !setting.UI.SVG.Enabled {
+		opts.Disposition = "attachment"
+	}
+
+	ServeSetHeaders(w, opts)
+}
+
+const mimeDetectionBufferLen = 1024
+
+func ServeContentByReader(r *http.Request, w http.ResponseWriter, filePath string, size int64, reader io.Reader) {
+	buf := make([]byte, mimeDetectionBufferLen)
+	n, err := util.ReadAtMost(reader, buf)
+	if err != nil {
+		http.Error(w, "serve content: unable to pre-read", http.StatusRequestedRangeNotSatisfiable)
+		return
+	}
+	if n >= 0 {
+		buf = buf[:n]
+	}
+	setServeHeadersByFile(r, w, filePath, buf)
+
+	// reset the reader to the beginning
+	reader = io.MultiReader(bytes.NewReader(buf), reader)
+
+	rangeHeader := r.Header.Get("Range")
+
+	// if no size or no supported range, serve as 200 (complete response)
+	if size <= 0 || !strings.HasPrefix(rangeHeader, "bytes=") {
+		if size >= 0 {
+			w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
+		}
+		_, _ = io.Copy(w, reader) // just like http.ServeContent, not necessary to handle the error
+		return
+	}
+
+	// do our best to support the minimal "Range" request (no support for multiple range: "Range: bytes=0-50, 100-150")
+	//
+	// GET /...
+	// Range: bytes=0-1023
+	//
+	// HTTP/1.1 206 Partial Content
+	// Content-Range: bytes 0-1023/146515
+	// Content-Length: 1024
+
+	_, rangeParts, _ := strings.Cut(rangeHeader, "=")
+	rangeBytesStart, rangeBytesEnd, found := strings.Cut(rangeParts, "-")
+	start, err := strconv.ParseInt(rangeBytesStart, 10, 64)
+	if start < 0 || start >= size {
+		err = errors.New("invalid start range")
+	}
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable)
+		return
+	}
+	end, err := strconv.ParseInt(rangeBytesEnd, 10, 64)
+	if rangeBytesEnd == "" && found {
+		err = nil
+		end = size - 1
+	}
+	if end >= size {
+		end = size - 1
+	}
+	if end < start {
+		err = errors.New("invalid end range")
+	}
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	partialLength := end - start + 1
+	w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, size))
+	w.Header().Set("Content-Length", strconv.FormatInt(partialLength, 10))
+	if _, err = io.CopyN(io.Discard, reader, start); err != nil {
+		http.Error(w, "serve content: unable to skip", http.StatusInternalServerError)
+		return
+	}
+
+	w.WriteHeader(http.StatusPartialContent)
+	_, _ = io.CopyN(w, reader, partialLength) // just like http.ServeContent, not necessary to handle the error
+}
+
+func ServeContentByReadSeeker(r *http.Request, w http.ResponseWriter, filePath string, modTime time.Time, reader io.ReadSeeker) {
+	buf := make([]byte, mimeDetectionBufferLen)
+	n, err := util.ReadAtMost(reader, buf)
+	if err != nil {
+		http.Error(w, "serve content: unable to read", http.StatusInternalServerError)
+		return
+	}
+	if _, err = reader.Seek(0, io.SeekStart); err != nil {
+		http.Error(w, "serve content: unable to seek", http.StatusInternalServerError)
+		return
+	}
+	if n >= 0 {
+		buf = buf[:n]
+	}
+	setServeHeadersByFile(r, w, filePath, buf)
+	http.ServeContent(w, r, path.Base(filePath), modTime, reader)
+}
diff --git a/modules/httplib/serve_test.go b/modules/httplib/serve_test.go
new file mode 100644
index 0000000000..0768f1c713
--- /dev/null
+++ b/modules/httplib/serve_test.go
@@ -0,0 +1,109 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package httplib
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestServeContentByReader(t *testing.T) {
+	data := "0123456789abcdef"
+
+	test := func(t *testing.T, expectedStatusCode int, expectedContent string) {
+		_, rangeStr, _ := strings.Cut(t.Name(), "_range_")
+		r := &http.Request{Header: http.Header{}, Form: url.Values{}}
+		if rangeStr != "" {
+			r.Header.Set("Range", fmt.Sprintf("bytes=%s", rangeStr))
+		}
+		reader := strings.NewReader(data)
+		w := NewMockResponseWriter()
+		ServeContentByReader(r, w, "test", int64(len(data)), reader)
+		assert.Equal(t, expectedStatusCode, w.StatusCode)
+		if expectedStatusCode == http.StatusPartialContent || expectedStatusCode == http.StatusOK {
+			assert.Equal(t, fmt.Sprint(len(expectedContent)), w.Header().Get("Content-Length"))
+			assert.Equal(t, expectedContent, w.BodyBuffer.String())
+		}
+	}
+
+	t.Run("_range_", func(t *testing.T) {
+		test(t, http.StatusOK, data)
+	})
+	t.Run("_range_0-", func(t *testing.T) {
+		test(t, http.StatusPartialContent, data)
+	})
+	t.Run("_range_0-15", func(t *testing.T) {
+		test(t, http.StatusPartialContent, data)
+	})
+	t.Run("_range_1-", func(t *testing.T) {
+		test(t, http.StatusPartialContent, data[1:])
+	})
+	t.Run("_range_1-3", func(t *testing.T) {
+		test(t, http.StatusPartialContent, data[1:3+1])
+	})
+	t.Run("_range_16-", func(t *testing.T) {
+		test(t, http.StatusRequestedRangeNotSatisfiable, "")
+	})
+	t.Run("_range_1-99999", func(t *testing.T) {
+		test(t, http.StatusPartialContent, data[1:])
+	})
+}
+
+func TestServeContentByReadSeeker(t *testing.T) {
+	data := "0123456789abcdef"
+	tmpFile := t.TempDir() + "/test"
+	err := os.WriteFile(tmpFile, []byte(data), 0o644)
+	assert.NoError(t, err)
+
+	test := func(t *testing.T, expectedStatusCode int, expectedContent string) {
+		_, rangeStr, _ := strings.Cut(t.Name(), "_range_")
+		r := &http.Request{Header: http.Header{}, Form: url.Values{}}
+		if rangeStr != "" {
+			r.Header.Set("Range", fmt.Sprintf("bytes=%s", rangeStr))
+		}
+
+		seekReader, err := os.OpenFile(tmpFile, os.O_RDONLY, 0o644)
+		if !assert.NoError(t, err) {
+			return
+		}
+		defer seekReader.Close()
+
+		w := NewMockResponseWriter()
+		ServeContentByReadSeeker(r, w, "test", time.Time{}, seekReader)
+		assert.Equal(t, expectedStatusCode, w.StatusCode)
+		if expectedStatusCode == http.StatusPartialContent || expectedStatusCode == http.StatusOK {
+			assert.Equal(t, fmt.Sprint(len(expectedContent)), w.Header().Get("Content-Length"))
+			assert.Equal(t, expectedContent, w.BodyBuffer.String())
+		}
+	}
+
+	t.Run("_range_", func(t *testing.T) {
+		test(t, http.StatusOK, data)
+	})
+	t.Run("_range_0-", func(t *testing.T) {
+		test(t, http.StatusPartialContent, data)
+	})
+	t.Run("_range_0-15", func(t *testing.T) {
+		test(t, http.StatusPartialContent, data)
+	})
+	t.Run("_range_1-", func(t *testing.T) {
+		test(t, http.StatusPartialContent, data[1:])
+	})
+	t.Run("_range_1-3", func(t *testing.T) {
+		test(t, http.StatusPartialContent, data[1:3+1])
+	})
+	t.Run("_range_16-", func(t *testing.T) {
+		test(t, http.StatusRequestedRangeNotSatisfiable, "")
+	})
+	t.Run("_range_1-99999", func(t *testing.T) {
+		test(t, http.StatusPartialContent, data[1:])
+	})
+}
diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go
index 53fac4ab85..daf8c6cfdd 100644
--- a/modules/lfs/content_store.go
+++ b/modules/lfs/content_store.go
@@ -18,9 +18,9 @@ import (
 
 var (
 	// ErrHashMismatch occurs if the content has does not match OID
-	ErrHashMismatch = errors.New("Content hash does not match OID")
+	ErrHashMismatch = errors.New("content hash does not match OID")
 	// ErrSizeMismatch occurs if the content size does not match
-	ErrSizeMismatch = errors.New("Content size does not match")
+	ErrSizeMismatch = errors.New("content size does not match")
 )
 
 // ContentStore provides a simple file system based storage.
@@ -105,7 +105,7 @@ func (s *ContentStore) Verify(pointer Pointer) (bool, error) {
 }
 
 // ReadMetaObject will read a git_model.LFSMetaObject and return a reader
-func ReadMetaObject(pointer Pointer) (io.ReadCloser, error) {
+func ReadMetaObject(pointer Pointer) (io.ReadSeekCloser, error) {
 	contentStore := NewContentStore()
 	return contentStore.Get(pointer)
 }