From 5727056ea109eb04ee535b981349cdfb44df9fae Mon Sep 17 00:00:00 2001
From: wxiaoguang <wxiaoguang@gmail.com>
Date: Tue, 28 Mar 2023 23:10:24 +0800
Subject: Make minio package support legacy MD5 checksum (#23768)

A feedback from discord:
https://discord.com/channels/322538954119184384/561007778139734027/1090185427115319386

Some storages like:

 * https://developers.cloudflare.com/r2/api/s3/api/
 * https://www.backblaze.com/b2/docs/s3_compatible_api.html

They do not support "x-amz-checksum-algorithm" header

But minio recently uses that header with CRC32C by default. So we have
to tell minio to use legacy MD5 checksum.

I guess this needs to be backported because IIRC we 1.19 and 1.20 are
using similar minio package.


The minio package code for SendContentMD5 looks like this:

<details>

<img width="755" alt="image"
src="https://user-images.githubusercontent.com/2114189/228186768-4f2f6f67-62b9-4aee-9251-5af714ad9674.png">

</details>
---
 modules/storage/minio.go | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

(limited to 'modules/storage')

diff --git a/modules/storage/minio.go b/modules/storage/minio.go
index 5c67dbf26a..250f17827f 100644
--- a/modules/storage/minio.go
+++ b/modules/storage/minio.go
@@ -6,6 +6,7 @@ package storage
 import (
 	"context"
 	"crypto/tls"
+	"fmt"
 	"io"
 	"net/http"
 	"net/url"
@@ -53,10 +54,12 @@ type MinioStorageConfig struct {
 	BasePath           string `ini:"MINIO_BASE_PATH"`
 	UseSSL             bool   `ini:"MINIO_USE_SSL"`
 	InsecureSkipVerify bool   `ini:"MINIO_INSECURE_SKIP_VERIFY"`
+	ChecksumAlgorithm  string `ini:"MINIO_CHECKSUM_ALGORITHM"`
 }
 
 // MinioStorage returns a minio bucket storage
 type MinioStorage struct {
+	cfg      *MinioStorageConfig
 	ctx      context.Context
 	client   *minio.Client
 	bucket   string
@@ -91,6 +94,10 @@ func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error
 	}
 	config := configInterface.(MinioStorageConfig)
 
+	if config.ChecksumAlgorithm != "" && config.ChecksumAlgorithm != "default" && config.ChecksumAlgorithm != "md5" {
+		return nil, fmt.Errorf("invalid minio checksum algorithm: %s", config.ChecksumAlgorithm)
+	}
+
 	log.Info("Creating Minio storage at %s:%s with base path %s", config.Endpoint, config.Bucket, config.BasePath)
 
 	minioClient, err := minio.New(config.Endpoint, &minio.Options{
@@ -113,6 +120,7 @@ func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error
 	}
 
 	return &MinioStorage{
+		cfg:      &config,
 		ctx:      ctx,
 		client:   minioClient,
 		bucket:   config.Bucket,
@@ -124,7 +132,7 @@ func (m *MinioStorage) buildMinioPath(p string) string {
 	return util.PathJoinRelX(m.basePath, p)
 }
 
-// Open open a file
+// Open opens a file
 func (m *MinioStorage) Open(path string) (Object, error) {
 	opts := minio.GetObjectOptions{}
 	object, err := m.client.GetObject(m.ctx, m.bucket, m.buildMinioPath(path), opts)
@@ -134,7 +142,7 @@ func (m *MinioStorage) Open(path string) (Object, error) {
 	return &minioObject{object}, nil
 }
 
-// Save save a file to minio
+// Save saves a file to minio
 func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error) {
 	uploadInfo, err := m.client.PutObject(
 		m.ctx,
@@ -142,7 +150,14 @@ func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error)
 		m.buildMinioPath(path),
 		r,
 		size,
-		minio.PutObjectOptions{ContentType: "application/octet-stream"},
+		minio.PutObjectOptions{
+			ContentType: "application/octet-stream",
+			// some storages like:
+			// * https://developers.cloudflare.com/r2/api/s3/api/
+			// * https://www.backblaze.com/b2/docs/s3_compatible_api.html
+			// do not support "x-amz-checksum-algorithm" header, so use legacy MD5 checksum
+			SendContentMd5: m.cfg.ChecksumAlgorithm == "md5",
+		},
 	)
 	if err != nil {
 		return 0, convertMinioErr(err)
-- 
cgit v1.2.3