From 62e6c9bc6c7a94a02a263b40e78a4563788e7bc3 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 18 Aug 2020 12:23:45 +0800 Subject: Add a storage layer for attachments (#11387) * Add a storage layer for attachments * Fix some bug * fix test * Fix copyright head and lint * Fix bug * Add setting for minio and flags for migrate-storage * Add documents * fix lint * Add test for minio store type on attachments * fix test * fix test * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Add warning when storage migrated successfully * Fix drone * fix test * rebase * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * remove log on xorm * Fi download bug * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * Add URL function to serve attachments directly from S3/Minio * Add ability to enable/disable redirection in attachment configuration * Fix typo * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * don't change unrelated files * Fix lint * Fix build * update go.mod and go.sum * Use github.com/minio/minio-go/v6 * Remove unused function * Upgrade minio to v7 and some other improvements * fix lint * Fix go mod Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> Co-authored-by: Tyler --- modules/storage/minio.go | 101 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 modules/storage/minio.go (limited to 'modules/storage/minio.go') diff --git a/modules/storage/minio.go b/modules/storage/minio.go new file mode 100644 index 0000000000..77d24e6b73 --- /dev/null +++ b/modules/storage/minio.go @@ -0,0 +1,101 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package storage + +import ( + "context" + "io" + "net/url" + "path" + "strings" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +var ( + _ ObjectStorage = &MinioStorage{} + quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"") +) + +// MinioStorage returns a minio bucket storage +type MinioStorage struct { + ctx context.Context + client *minio.Client + bucket string + basePath string +} + +// NewMinioStorage returns a minio storage +func NewMinioStorage(ctx context.Context, endpoint, accessKeyID, secretAccessKey, bucket, location, basePath string, useSSL bool) (*MinioStorage, error) { + minioClient, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: useSSL, + }) + if err != nil { + return nil, err + } + + if err := minioClient.MakeBucket(ctx, bucket, minio.MakeBucketOptions{ + Region: location, + }); err != nil { + // Check to see if we already own this bucket (which happens if you run this twice) + exists, errBucketExists := minioClient.BucketExists(ctx, bucket) + if !exists || errBucketExists != nil { + return nil, err + } + } + + return &MinioStorage{ + ctx: ctx, + client: minioClient, + bucket: bucket, + basePath: basePath, + }, nil +} + +func (m *MinioStorage) buildMinioPath(p string) string { + return strings.TrimPrefix(path.Join(m.basePath, p), "/") +} + +// Open open a file +func (m *MinioStorage) Open(path string) (io.ReadCloser, error) { + var opts = minio.GetObjectOptions{} + object, err := m.client.GetObject(m.ctx, m.bucket, m.buildMinioPath(path), opts) + if err != nil { + return nil, err + } + return object, nil +} + +// Save save a file to minio +func (m *MinioStorage) Save(path string, r io.Reader) (int64, error) { + uploadInfo, err := m.client.PutObject( + m.ctx, + m.bucket, + m.buildMinioPath(path), + r, + -1, + minio.PutObjectOptions{ContentType: "application/octet-stream"}, + ) + if err != nil { + return 0, err + } + return uploadInfo.Size, nil +} + +// Delete delete a file +func (m *MinioStorage) Delete(path string) error { + return m.client.RemoveObject(m.ctx, m.bucket, m.buildMinioPath(path), minio.RemoveObjectOptions{}) +} + +// URL gets the redirect URL to a file. The presigned link is valid for 5 minutes. +func (m *MinioStorage) URL(path, name string) (*url.URL, error) { + reqParams := make(url.Values) + // TODO it may be good to embed images with 'inline' like ServeData does, but we don't want to have to read the file, do we? + reqParams.Set("response-content-disposition", "attachment; filename=\""+quoteEscaper.Replace(name)+"\"") + return m.client.PresignedGetObject(m.ctx, m.bucket, m.buildMinioPath(path), 5*time.Minute, reqParams) +} -- cgit v1.2.3