123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102 |
- // Copyright 2022 The Gitea Authors. All rights reserved.
- // SPDX-License-Identifier: MIT
-
- package dbfs
-
- import (
- "context"
- "os"
-
- "code.gitea.io/gitea/models/db"
- )
-
- /*
- The reasons behind the DBFS (database-filesystem) package:
- When a Gitea action is running, the Gitea action server should collect and store all the logs.
-
- The requirements are:
- * The running logs must be stored across the cluster if the Gitea servers are deployed as a cluster.
- * The logs will be archived to Object Storage (S3/MinIO, etc.) after a period of time.
- * The Gitea action UI should be able to render the running logs and the archived logs.
-
- Some possible solutions for the running logs:
- * [Not ideal] Using local temp file: it can not be shared across the cluster.
- * [Not ideal] Using shared file in the filesystem of git repository: although at the moment, the Gitea cluster's
- git repositories must be stored in a shared filesystem, in the future, Gitea may need a dedicated Git Service Server
- to decouple the shared filesystem. Then the action logs will become a blocker.
- * [Not ideal] Record the logs in a database table line by line: it has a couple of problems:
- - It's difficult to make multiple increasing sequence (log line number) for different databases.
- - The database table will have a lot of rows and be affected by the big-table performance problem.
- - It's difficult to load logs by using the same interface as other storages.
- - It's difficult to calculate the size of the logs.
-
- The DBFS solution:
- * It can be used in a cluster.
- * It can share the same interface (Read/Write/Seek) as other storages.
- * It's very friendly to database because it only needs to store much fewer rows than the log-line solution.
- * In the future, when Gitea action needs to limit the log size (other CI/CD services also do so), it's easier to calculate the log file size.
- * Even sometimes the UI needs to render the tailing lines, the tailing lines can be found be counting the "\n" from the end of the file by seek.
- The seeking and finding is not the fastest way, but it's still acceptable and won't affect the performance too much.
- */
-
- type dbfsMeta struct {
- ID int64 `xorm:"pk autoincr"`
- FullPath string `xorm:"VARCHAR(500) UNIQUE NOT NULL"`
- BlockSize int64 `xorm:"BIGINT NOT NULL"`
- FileSize int64 `xorm:"BIGINT NOT NULL"`
- CreateTimestamp int64 `xorm:"BIGINT NOT NULL"`
- ModifyTimestamp int64 `xorm:"BIGINT NOT NULL"`
- }
-
- type dbfsData struct {
- ID int64 `xorm:"pk autoincr"`
- Revision int64 `xorm:"BIGINT NOT NULL"`
- MetaID int64 `xorm:"BIGINT index(meta_offset) NOT NULL"`
- BlobOffset int64 `xorm:"BIGINT index(meta_offset) NOT NULL"`
- BlobSize int64 `xorm:"BIGINT NOT NULL"`
- BlobData []byte `xorm:"BLOB NOT NULL"`
- }
-
- func init() {
- db.RegisterModel(new(dbfsMeta))
- db.RegisterModel(new(dbfsData))
- }
-
- func OpenFile(ctx context.Context, name string, flag int) (File, error) {
- f, err := newDbFile(ctx, name)
- if err != nil {
- return nil, err
- }
- err = f.open(flag)
- if err != nil {
- _ = f.Close()
- return nil, err
- }
- return f, nil
- }
-
- func Open(ctx context.Context, name string) (File, error) {
- return OpenFile(ctx, name, os.O_RDONLY)
- }
-
- func Create(ctx context.Context, name string) (File, error) {
- return OpenFile(ctx, name, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
- }
-
- func Rename(ctx context.Context, oldPath, newPath string) error {
- f, err := newDbFile(ctx, oldPath)
- if err != nil {
- return err
- }
- defer f.Close()
- return f.renameTo(newPath)
- }
-
- func Remove(ctx context.Context, name string) error {
- f, err := newDbFile(ctx, name)
- if err != nil {
- return err
- }
- defer f.Close()
- return f.delete()
- }
|