diff options
author | wxiaoguang <wxiaoguang@gmail.com> | 2023-03-28 23:10:24 +0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-03-28 11:10:24 -0400 |
commit | 5727056ea109eb04ee535b981349cdfb44df9fae (patch) | |
tree | af2b96f7b7871dd3d07169642b6234369c032632 /modules/storage | |
parent | 6a0ef71984b9246fc1bb378caaa614f3dedaa9e9 (diff) | |
download | gitea-5727056ea109eb04ee535b981349cdfb44df9fae.tar.gz gitea-5727056ea109eb04ee535b981349cdfb44df9fae.zip |
Make minio package support legacy MD5 checksum (#23768)
A feedback from discord:
https://discord.com/channels/322538954119184384/561007778139734027/1090185427115319386
Some storages like:
* https://developers.cloudflare.com/r2/api/s3/api/
* https://www.backblaze.com/b2/docs/s3_compatible_api.html
They do not support "x-amz-checksum-algorithm" header
But minio recently uses that header with CRC32C by default. So we have
to tell minio to use legacy MD5 checksum.
I guess this needs to be backported because IIRC we 1.19 and 1.20 are
using similar minio package.
The minio package code for SendContentMD5 looks like this:
<details>
<img width="755" alt="image"
src="https://user-images.githubusercontent.com/2114189/228186768-4f2f6f67-62b9-4aee-9251-5af714ad9674.png">
</details>
Diffstat (limited to 'modules/storage')
-rw-r--r-- | modules/storage/minio.go | 21 |
1 files changed, 18 insertions, 3 deletions
diff --git a/modules/storage/minio.go b/modules/storage/minio.go index 5c67dbf26a..250f17827f 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -6,6 +6,7 @@ package storage import ( "context" "crypto/tls" + "fmt" "io" "net/http" "net/url" @@ -53,10 +54,12 @@ type MinioStorageConfig struct { BasePath string `ini:"MINIO_BASE_PATH"` UseSSL bool `ini:"MINIO_USE_SSL"` InsecureSkipVerify bool `ini:"MINIO_INSECURE_SKIP_VERIFY"` + ChecksumAlgorithm string `ini:"MINIO_CHECKSUM_ALGORITHM"` } // MinioStorage returns a minio bucket storage type MinioStorage struct { + cfg *MinioStorageConfig ctx context.Context client *minio.Client bucket string @@ -91,6 +94,10 @@ func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error } config := configInterface.(MinioStorageConfig) + if config.ChecksumAlgorithm != "" && config.ChecksumAlgorithm != "default" && config.ChecksumAlgorithm != "md5" { + return nil, fmt.Errorf("invalid minio checksum algorithm: %s", config.ChecksumAlgorithm) + } + log.Info("Creating Minio storage at %s:%s with base path %s", config.Endpoint, config.Bucket, config.BasePath) minioClient, err := minio.New(config.Endpoint, &minio.Options{ @@ -113,6 +120,7 @@ func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error } return &MinioStorage{ + cfg: &config, ctx: ctx, client: minioClient, bucket: config.Bucket, @@ -124,7 +132,7 @@ func (m *MinioStorage) buildMinioPath(p string) string { return util.PathJoinRelX(m.basePath, p) } -// Open open a file +// Open opens a file func (m *MinioStorage) Open(path string) (Object, error) { opts := minio.GetObjectOptions{} object, err := m.client.GetObject(m.ctx, m.bucket, m.buildMinioPath(path), opts) @@ -134,7 +142,7 @@ func (m *MinioStorage) Open(path string) (Object, error) { return &minioObject{object}, nil } -// Save save a file to minio +// Save saves a file to minio func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error) { uploadInfo, err := m.client.PutObject( m.ctx, @@ -142,7 +150,14 @@ func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error) m.buildMinioPath(path), r, size, - minio.PutObjectOptions{ContentType: "application/octet-stream"}, + minio.PutObjectOptions{ + ContentType: "application/octet-stream", + // some storages like: + // * https://developers.cloudflare.com/r2/api/s3/api/ + // * https://www.backblaze.com/b2/docs/s3_compatible_api.html + // do not support "x-amz-checksum-algorithm" header, so use legacy MD5 checksum + SendContentMd5: m.cfg.ChecksumAlgorithm == "md5", + }, ) if err != nil { return 0, convertMinioErr(err) |