Backport #23768 (no source code conflict, only some unrelated docs/test-ini conflicts) Some storages like: * https://developers.cloudflare.com/r2/api/s3/api/ * https://www.backblaze.com/b2/docs/s3_compatible_api.html They do not support "x-amz-checksum-algorithm" header But minio recently uses that header with CRC32C by default. So we have to tell minio to use legacy MD5 checksum.tags/v1.19.1
@@ -72,12 +72,21 @@ var CmdMigrateStorage = cli.Command{ | |||
cli.StringFlag{ | |||
Name: "minio-base-path", | |||
Value: "", | |||
Usage: "Minio storage basepath on the bucket", | |||
Usage: "Minio storage base path on the bucket", | |||
}, | |||
cli.BoolFlag{ | |||
Name: "minio-use-ssl", | |||
Usage: "Enable SSL for minio", | |||
}, | |||
cli.BoolFlag{ | |||
Name: "minio-insecure-skip-verify", | |||
Usage: "Skip SSL verification", | |||
}, | |||
cli.StringFlag{ | |||
Name: "minio-checksum-algorithm", | |||
Value: "", | |||
Usage: "Minio checksum algorithm (default/md5)", | |||
}, | |||
}, | |||
} | |||
@@ -168,13 +177,15 @@ func runMigrateStorage(ctx *cli.Context) error { | |||
dstStorage, err = storage.NewMinioStorage( | |||
stdCtx, | |||
storage.MinioStorageConfig{ | |||
Endpoint: ctx.String("minio-endpoint"), | |||
AccessKeyID: ctx.String("minio-access-key-id"), | |||
SecretAccessKey: ctx.String("minio-secret-access-key"), | |||
Bucket: ctx.String("minio-bucket"), | |||
Location: ctx.String("minio-location"), | |||
BasePath: ctx.String("minio-base-path"), | |||
UseSSL: ctx.Bool("minio-use-ssl"), | |||
Endpoint: ctx.String("minio-endpoint"), | |||
AccessKeyID: ctx.String("minio-access-key-id"), | |||
SecretAccessKey: ctx.String("minio-secret-access-key"), | |||
Bucket: ctx.String("minio-bucket"), | |||
Location: ctx.String("minio-location"), | |||
BasePath: ctx.String("minio-base-path"), | |||
UseSSL: ctx.Bool("minio-use-ssl"), | |||
InsecureSkipVerify: ctx.Bool("minio-insecure-skip-verify"), | |||
ChecksumAlgorithm: ctx.String("minio-checksum-algorithm"), | |||
}) | |||
default: | |||
return fmt.Errorf("unsupported storage type: %s", ctx.String("storage")) |
@@ -1874,6 +1874,9 @@ ROUTER = console | |||
;; | |||
;; Minio skip SSL verification available when STORAGE_TYPE is `minio` | |||
;MINIO_INSECURE_SKIP_VERIFY = false | |||
;; | |||
;; Minio checksum algorithm: default (for MinIO or AWS S3) or md5 (for Cloudflare or Backblaze) | |||
;MINIO_CHECKSUM_ALGORITHM = default | |||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; | |||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; |
@@ -855,6 +855,7 @@ Default templates for project boards: | |||
- `MINIO_BASE_PATH`: **attachments/**: Minio base path on the bucket only available when STORAGE_TYPE is `minio` | |||
- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when STORAGE_TYPE is `minio` | |||
- `MINIO_INSECURE_SKIP_VERIFY`: **false**: Minio skip SSL verification available when STORAGE_TYPE is `minio` | |||
- `MINIO_CHECKSUM_ALGORITHM`: **default**: Minio checksum algorithm: `default` (for MinIO or AWS S3) or `md5` (for Cloudflare or Backblaze) | |||
## Log (`log`) | |||
@@ -59,15 +59,22 @@ func (s *ContentStore) Put(pointer Pointer, r io.Reader) error { | |||
return err | |||
} | |||
// This shouldn't happen but it is sensible to test | |||
if written != pointer.Size { | |||
if err := s.Delete(p); err != nil { | |||
log.Error("Cleaning the LFS OID[%s] failed: %v", pointer.Oid, err) | |||
// check again whether there is any error during the Save operation | |||
// because some errors might be ignored by the Reader's caller | |||
if wrappedRd.lastError != nil && !errors.Is(wrappedRd.lastError, io.EOF) { | |||
err = wrappedRd.lastError | |||
} else if written != pointer.Size { | |||
err = ErrSizeMismatch | |||
} | |||
// if the upload failed, try to delete the file | |||
if err != nil { | |||
if errDel := s.Delete(p); errDel != nil { | |||
log.Error("Cleaning the LFS OID[%s] failed: %v", pointer.Oid, errDel) | |||
} | |||
return ErrSizeMismatch | |||
} | |||
return nil | |||
return err | |||
} | |||
// Exists returns true if the object exists in the content store. | |||
@@ -108,6 +115,17 @@ type hashingReader struct { | |||
expectedSize int64 | |||
hash hash.Hash | |||
expectedHash string | |||
lastError error | |||
} | |||
// recordError records the last error during the Save operation | |||
// Some callers of the Reader doesn't respect the returned "err" | |||
// For example, MinIO's Put will ignore errors if the written size could equal to expected size | |||
// So we must remember the error by ourselves, | |||
// and later check again whether ErrSizeMismatch or ErrHashMismatch occurs during the Save operation | |||
func (r *hashingReader) recordError(err error) error { | |||
r.lastError = err | |||
return err | |||
} | |||
func (r *hashingReader) Read(b []byte) (int, error) { | |||
@@ -117,22 +135,22 @@ func (r *hashingReader) Read(b []byte) (int, error) { | |||
r.currentSize += int64(n) | |||
wn, werr := r.hash.Write(b[:n]) | |||
if wn != n || werr != nil { | |||
return n, werr | |||
return n, r.recordError(werr) | |||
} | |||
} | |||
if err != nil && err == io.EOF { | |||
if errors.Is(err, io.EOF) || r.currentSize >= r.expectedSize { | |||
if r.currentSize != r.expectedSize { | |||
return n, ErrSizeMismatch | |||
return n, r.recordError(ErrSizeMismatch) | |||
} | |||
shaStr := hex.EncodeToString(r.hash.Sum(nil)) | |||
if shaStr != r.expectedHash { | |||
return n, ErrHashMismatch | |||
return n, r.recordError(ErrHashMismatch) | |||
} | |||
} | |||
return n, err | |||
return n, r.recordError(err) | |||
} | |||
func newHashingReader(expectedSize int64, expectedHash string, reader io.Reader) *hashingReader { |
@@ -42,6 +42,7 @@ func getStorage(rootCfg ConfigProvider, name, typ string, targetSec *ini.Section | |||
sec.Key("MINIO_LOCATION").MustString("us-east-1") | |||
sec.Key("MINIO_USE_SSL").MustBool(false) | |||
sec.Key("MINIO_INSECURE_SKIP_VERIFY").MustBool(false) | |||
sec.Key("MINIO_CHECKSUM_ALGORITHM").MustString("default") | |||
if targetSec == nil { | |||
targetSec, _ = rootCfg.NewSection(name) |
@@ -6,6 +6,7 @@ package storage | |||
import ( | |||
"context" | |||
"crypto/tls" | |||
"fmt" | |||
"io" | |||
"net/http" | |||
"net/url" | |||
@@ -52,10 +53,12 @@ type MinioStorageConfig struct { | |||
BasePath string `ini:"MINIO_BASE_PATH"` | |||
UseSSL bool `ini:"MINIO_USE_SSL"` | |||
InsecureSkipVerify bool `ini:"MINIO_INSECURE_SKIP_VERIFY"` | |||
ChecksumAlgorithm string `ini:"MINIO_CHECKSUM_ALGORITHM"` | |||
} | |||
// MinioStorage returns a minio bucket storage | |||
type MinioStorage struct { | |||
cfg *MinioStorageConfig | |||
ctx context.Context | |||
client *minio.Client | |||
bucket string | |||
@@ -90,6 +93,10 @@ func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error | |||
} | |||
config := configInterface.(MinioStorageConfig) | |||
if config.ChecksumAlgorithm != "" && config.ChecksumAlgorithm != "default" && config.ChecksumAlgorithm != "md5" { | |||
return nil, fmt.Errorf("invalid minio checksum algorithm: %s", config.ChecksumAlgorithm) | |||
} | |||
log.Info("Creating Minio storage at %s:%s with base path %s", config.Endpoint, config.Bucket, config.BasePath) | |||
minioClient, err := minio.New(config.Endpoint, &minio.Options{ | |||
@@ -112,6 +119,7 @@ func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error | |||
} | |||
return &MinioStorage{ | |||
cfg: &config, | |||
ctx: ctx, | |||
client: minioClient, | |||
bucket: config.Bucket, | |||
@@ -123,7 +131,7 @@ func (m *MinioStorage) buildMinioPath(p string) string { | |||
return strings.TrimPrefix(path.Join(m.basePath, path.Clean("/" + strings.ReplaceAll(p, "\\", "/"))[1:]), "/") | |||
} | |||
// Open open a file | |||
// Open opens a file | |||
func (m *MinioStorage) Open(path string) (Object, error) { | |||
opts := minio.GetObjectOptions{} | |||
object, err := m.client.GetObject(m.ctx, m.bucket, m.buildMinioPath(path), opts) | |||
@@ -133,7 +141,7 @@ func (m *MinioStorage) Open(path string) (Object, error) { | |||
return &minioObject{object}, nil | |||
} | |||
// Save save a file to minio | |||
// Save saves a file to minio | |||
func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error) { | |||
uploadInfo, err := m.client.PutObject( | |||
m.ctx, | |||
@@ -141,7 +149,14 @@ func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error) | |||
m.buildMinioPath(path), | |||
r, | |||
size, | |||
minio.PutObjectOptions{ContentType: "application/octet-stream"}, | |||
minio.PutObjectOptions{ | |||
ContentType: "application/octet-stream", | |||
// some storages like: | |||
// * https://developers.cloudflare.com/r2/api/s3/api/ | |||
// * https://www.backblaze.com/b2/docs/s3_compatible_api.html | |||
// do not support "x-amz-checksum-algorithm" header, so use legacy MD5 checksum | |||
SendContentMd5: m.cfg.ChecksumAlgorithm == "md5", | |||
}, | |||
) | |||
if err != nil { | |||
return 0, convertMinioErr(err) |
@@ -76,6 +76,7 @@ MINIO_SECRET_ACCESS_KEY = 12345678 | |||
MINIO_BUCKET = gitea | |||
MINIO_LOCATION = us-east-1 | |||
MINIO_USE_SSL = false | |||
MINIO_CHECKSUM_ALGORITHM = md5 | |||
[mailer] | |||
ENABLED = true |