* Fix minio bug * Add tests for storage configuration * Change the Seek flag to keep compitable minio? * Fix test when first-byte-pos of all ranges is greater than the resource length Co-authored-by: techknowlogick <techknowlogick@gitea.io> Co-authored-by: techknowlogick <techknowlogick@gitea.io>tags/v1.13.0-rc2
@@ -11,7 +11,6 @@ import ( | |||
"encoding/json" | |||
"fmt" | |||
"io" | |||
"log" | |||
"net/http" | |||
"net/http/cookiejar" | |||
"net/http/httptest" | |||
@@ -27,8 +26,10 @@ import ( | |||
"code.gitea.io/gitea/models" | |||
"code.gitea.io/gitea/modules/base" | |||
"code.gitea.io/gitea/modules/graceful" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/queue" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/modules/storage" | |||
"code.gitea.io/gitea/modules/util" | |||
"code.gitea.io/gitea/routers" | |||
"code.gitea.io/gitea/routers/routes" | |||
@@ -59,6 +60,8 @@ func NewNilResponseRecorder() *NilResponseRecorder { | |||
} | |||
func TestMain(m *testing.M) { | |||
defer log.Close() | |||
managerCtx, cancel := context.WithCancel(context.Background()) | |||
graceful.InitManager(managerCtx) | |||
defer cancel() | |||
@@ -142,6 +145,10 @@ func initIntegrationTest() { | |||
util.RemoveAll(models.LocalCopyPath()) | |||
setting.CheckLFSVersion() | |||
setting.InitDBConfig() | |||
if err := storage.Init(); err != nil { | |||
fmt.Printf("Init storage failed: %v", err) | |||
os.Exit(1) | |||
} | |||
switch { | |||
case setting.Database.UseMySQL: | |||
@@ -149,27 +156,27 @@ func initIntegrationTest() { | |||
setting.Database.User, setting.Database.Passwd, setting.Database.Host)) | |||
defer db.Close() | |||
if err != nil { | |||
log.Fatalf("sql.Open: %v", err) | |||
log.Fatal("sql.Open: %v", err) | |||
} | |||
if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", setting.Database.Name)); err != nil { | |||
log.Fatalf("db.Exec: %v", err) | |||
log.Fatal("db.Exec: %v", err) | |||
} | |||
case setting.Database.UsePostgreSQL: | |||
db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s", | |||
setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.SSLMode)) | |||
defer db.Close() | |||
if err != nil { | |||
log.Fatalf("sql.Open: %v", err) | |||
log.Fatal("sql.Open: %v", err) | |||
} | |||
dbrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", setting.Database.Name)) | |||
if err != nil { | |||
log.Fatalf("db.Query: %v", err) | |||
log.Fatal("db.Query: %v", err) | |||
} | |||
defer dbrows.Close() | |||
if !dbrows.Next() { | |||
if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", setting.Database.Name)); err != nil { | |||
log.Fatalf("db.Exec: CREATE DATABASE: %v", err) | |||
log.Fatal("db.Exec: CREATE DATABASE: %v", err) | |||
} | |||
} | |||
// Check if we need to setup a specific schema | |||
@@ -183,18 +190,18 @@ func initIntegrationTest() { | |||
// This is a different db object; requires a different Close() | |||
defer db.Close() | |||
if err != nil { | |||
log.Fatalf("sql.Open: %v", err) | |||
log.Fatal("sql.Open: %v", err) | |||
} | |||
schrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM information_schema.schemata WHERE schema_name = '%s'", setting.Database.Schema)) | |||
if err != nil { | |||
log.Fatalf("db.Query: %v", err) | |||
log.Fatal("db.Query: %v", err) | |||
} | |||
defer schrows.Close() | |||
if !schrows.Next() { | |||
// Create and setup a DB schema | |||
if _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", setting.Database.Schema)); err != nil { | |||
log.Fatalf("db.Exec: CREATE SCHEMA: %v", err) | |||
log.Fatal("db.Exec: CREATE SCHEMA: %v", err) | |||
} | |||
} | |||
@@ -203,10 +210,10 @@ func initIntegrationTest() { | |||
db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", | |||
host, port, "master", setting.Database.User, setting.Database.Passwd)) | |||
if err != nil { | |||
log.Fatalf("sql.Open: %v", err) | |||
log.Fatal("sql.Open: %v", err) | |||
} | |||
if _, err := db.Exec(fmt.Sprintf("If(db_id(N'%s') IS NULL) BEGIN CREATE DATABASE %s; END;", setting.Database.Name, setting.Database.Name)); err != nil { | |||
log.Fatalf("db.Exec: %v", err) | |||
log.Fatal("db.Exec: %v", err) | |||
} | |||
defer db.Close() | |||
} |
@@ -78,6 +78,7 @@ func storeAndGetLfs(t *testing.T, content *[]byte, extraHeader *http.Header, exp | |||
} | |||
} | |||
} | |||
resp := session.MakeRequest(t, req, expectedStatus) | |||
return resp | |||
@@ -210,7 +211,7 @@ func TestGetLFSRange(t *testing.T) { | |||
{"bytes=0-10", "123456789\n", http.StatusPartialContent}, | |||
// end-range bigger than length-1 is ignored | |||
{"bytes=0-11", "123456789\n", http.StatusPartialContent}, | |||
{"bytes=11-", "", http.StatusPartialContent}, | |||
{"bytes=11-", "Requested Range Not Satisfiable", http.StatusRequestedRangeNotSatisfiable}, | |||
// incorrect header value cause whole header to be ignored | |||
{"bytes=-", "123456789\n", http.StatusOK}, | |||
{"foobar", "123456789\n", http.StatusOK}, |
@@ -45,19 +45,21 @@ START_SSH_SERVER = true | |||
OFFLINE_MODE = false | |||
LFS_START_SERVER = true | |||
LFS_CONTENT_PATH = integrations/gitea-integration-mysql/datalfs-mysql | |||
LFS_JWT_SECRET = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w | |||
LFS_STORE_TYPE = minio | |||
LFS_SERVE_DIRECT = false | |||
LFS_MINIO_ENDPOINT = minio:9000 | |||
LFS_MINIO_ACCESS_KEY_ID = 123456 | |||
LFS_MINIO_SECRET_ACCESS_KEY = 12345678 | |||
LFS_MINIO_BUCKET = gitea | |||
LFS_MINIO_LOCATION = us-east-1 | |||
LFS_MINIO_BASE_PATH = lfs/ | |||
LFS_MINIO_USE_SSL = false | |||
[lfs] | |||
MINIO_BASE_PATH = lfs/ | |||
[attachment] | |||
MINIO_BASE_PATH = attachments/ | |||
[avatars] | |||
MINIO_BASE_PATH = avatars/ | |||
[repo-avatars] | |||
MINIO_BASE_PATH = repo-avatars/ | |||
[storage] | |||
STORAGE_TYPE = minio | |||
SERVE_DIRECT = false | |||
MINIO_ENDPOINT = minio:9000 | |||
@@ -65,7 +67,6 @@ MINIO_ACCESS_KEY_ID = 123456 | |||
MINIO_SECRET_ACCESS_KEY = 12345678 | |||
MINIO_BUCKET = gitea | |||
MINIO_LOCATION = us-east-1 | |||
MINIO_BASE_PATH = attachments/ | |||
MINIO_USE_SSL = false | |||
[mailer] | |||
@@ -88,9 +89,6 @@ ENABLE_NOTIFY_MAIL = true | |||
DISABLE_GRAVATAR = false | |||
ENABLE_FEDERATED_AVATAR = false | |||
AVATAR_UPLOAD_PATH = integrations/gitea-integration-mysql/data/avatars | |||
REPOSITORY_AVATAR_UPLOAD_PATH = integrations/gitea-integration-mysql/data/repo-avatars | |||
[session] | |||
PROVIDER = file | |||
PROVIDER_CONFIG = integrations/gitea-integration-mysql/data/sessions |
@@ -8,6 +8,7 @@ import ( | |||
"crypto/sha256" | |||
"encoding/hex" | |||
"errors" | |||
"fmt" | |||
"io" | |||
"os" | |||
@@ -21,6 +22,21 @@ var ( | |||
errSizeMismatch = errors.New("Content size does not match") | |||
) | |||
// ErrRangeNotSatisfiable represents an error which request range is not satisfiable. | |||
type ErrRangeNotSatisfiable struct { | |||
FromByte int64 | |||
} | |||
func (err ErrRangeNotSatisfiable) Error() string { | |||
return fmt.Sprintf("Requested range %d is not satisfiable", err.FromByte) | |||
} | |||
// IsErrRangeNotSatisfiable returns true if the error is an ErrRangeNotSatisfiable | |||
func IsErrRangeNotSatisfiable(err error) bool { | |||
_, ok := err.(ErrRangeNotSatisfiable) | |||
return ok | |||
} | |||
// ContentStore provides a simple file system based storage. | |||
type ContentStore struct { | |||
storage.ObjectStorage | |||
@@ -35,7 +51,12 @@ func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadC | |||
return nil, err | |||
} | |||
if fromByte > 0 { | |||
_, err = f.Seek(fromByte, os.SEEK_CUR) | |||
if fromByte >= meta.Size { | |||
return nil, ErrRangeNotSatisfiable{ | |||
FromByte: fromByte, | |||
} | |||
} | |||
_, err = f.Seek(fromByte, io.SeekStart) | |||
if err != nil { | |||
log.Error("Whilst trying to read LFS OID[%s]: Unable to seek to %d Error: %v", meta.Oid, fromByte, err) | |||
} |
@@ -191,8 +191,12 @@ func getContentHandler(ctx *context.Context) { | |||
contentStore := &ContentStore{ObjectStorage: storage.LFS} | |||
content, err := contentStore.Get(meta, fromByte) | |||
if err != nil { | |||
// Errors are logged in contentStore.Get | |||
writeStatus(ctx, 404) | |||
if IsErrRangeNotSatisfiable(err) { | |||
writeStatus(ctx, http.StatusRequestedRangeNotSatisfiable) | |||
} else { | |||
// Errors are logged in contentStore.Get | |||
writeStatus(ctx, 404) | |||
} | |||
return | |||
} | |||
defer content.Close() |
@@ -32,14 +32,12 @@ func (s *Storage) MapTo(v interface{}) error { | |||
} | |||
func getStorage(name, typ string, overrides ...*ini.Section) Storage { | |||
sectionName := "storage" | |||
if len(name) > 0 { | |||
sectionName = sectionName + "." + typ | |||
} | |||
const sectionName = "storage" | |||
sec := Cfg.Section(sectionName) | |||
if len(overrides) == 0 { | |||
overrides = []*ini.Section{ | |||
Cfg.Section(sectionName + "." + typ), | |||
Cfg.Section(sectionName + "." + name), | |||
} | |||
} |