feat: tranfer log to storage

modules/bots/log.go

@@ -14,6 +14,8 @@ import (
 	"time"
 
 	"code.gitea.io/gitea/models/dbfs"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/storage"
 	runnerv1 "gitea.com/gitea/proto-go/runner/v1"
 
 	"google.golang.org/protobuf/types/known/timestamppb"
@@ -55,15 +57,15 @@ func WriteLogs(ctx context.Context, filename string, offset int64, rows []*runne
 	return ns, nil
 }
 
-func ReadLogs(ctx context.Context, filename string, offset int64, limit int64) ([]*runnerv1.LogRow, error) {
-	name := DBFSPrefix + filename
-	f, err := dbfs.Open(ctx, name)
+func ReadLogs(ctx context.Context, inStorage bool, filename string, offset int64, limit int64) ([]*runnerv1.LogRow, error) {
+	f, err := openLogs(ctx, inStorage, filename)
 	if err != nil {
-		return nil, fmt.Errorf("dbfs Open %q: %w", name, err)
+		return nil, err
 	}
 	defer f.Close()
+
 	if _, err := f.Seek(offset, io.SeekStart); err != nil {
-		return nil, fmt.Errorf("dbfs Seek %q: %w", name, err)
+		return nil, fmt.Errorf("file seek: %w", err)
 	}
 
 	scanner := bufio.NewScanner(f)
@@ -89,6 +91,41 @@ func ReadLogs(ctx context.Context, filename string, offset int64, limit int64) (
 	return rows, nil
 }
 
+func TransferLogs(ctx context.Context, filename string) (func(), error) {
+	name := DBFSPrefix + filename
+	remove := func() {
+		if err := dbfs.Remove(ctx, name); err != nil {
+			log.Warn("dbfs remove %q: %v", name, err)
+		}
+	}
+	f, err := dbfs.Open(ctx, name)
+	if err != nil {
+		return nil, fmt.Errorf("dbfs open %q: %w", name, err)
+	}
+	defer f.Close()
+
+	if _, err := storage.Builds.Save(filename, f, -1); err != nil {
+		return nil, fmt.Errorf("storage save %q: %w", filename, err)
+	}
+	return remove, nil
+}
+
+func openLogs(ctx context.Context, inStorage bool, filename string) (io.ReadSeekCloser, error) {
+	if !inStorage {
+		name := DBFSPrefix + filename
+		f, err := dbfs.Open(ctx, name)
+		if err != nil {
+			return nil, fmt.Errorf("dbfs open %q: %w", name, err)
+		}
+		return f, nil
+	}
+	f, err := storage.Builds.Open(filename)
+	if err != nil {
+		return nil, fmt.Errorf("storage open %q: %w", filename, err)
+	}
+	return f, nil
+}
+
 func FormatLog(timestamp time.Time, content string) string {
 	// Content shouldn't contain new line, it will break log indexes, other control chars are safe.
 	content = strings.ReplaceAll(content, "\n", `\n`)

modules/setting/build.go

@@ -0,0 +1,28 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package setting
+
+import (
+	"code.gitea.io/gitea/modules/log"
+)
+
+// Builds settings
+var (
+	Builds = struct {
+		Storage
+		Enabled bool
+	}{
+		Enabled: true,
+	}
+)
+
+func newBuilds() {
+	sec := Cfg.Section("builds")
+	if err := sec.MapTo(&Builds); err != nil {
+		log.Fatal("Failed to map Builds settings: %v", err)
+	}
+
+	Builds.Storage = getStorage("builds", "", nil)
+}

modules/setting/setting.go

@@ -1062,6 +1062,8 @@ func loadFromConf(allowEmpty bool, extraConfig string) {
 
 	newPackages()
 
+	newBuilds()
+
 	if err = Cfg.Section("ui").MapTo(&UI); err != nil {
 		log.Fatal("Failed to map UI settings: %v", err)
 	} else if err = Cfg.Section("markdown").MapTo(&Markdown); err != nil {

modules/storage/storage.go

@@ -126,31 +126,27 @@ var (
 
 	// Packages represents packages storage
 	Packages ObjectStorage
+
+	// Builds represents builds storage
+	Builds ObjectStorage
 )
 
 // Init init the stoarge
 func Init() error {
-	if err := initAttachments(); err != nil {
-		return err
+	for _, f := range []func() error{
+		initAttachments,
+		initAvatars,
+		initRepoAvatars,
+		initLFS,
+		initRepoArchives,
+		initPackages,
+		initBuilds,
+	} {
+		if err := f(); err != nil {
+			return err
+		}
 	}
-
-	if err := initAvatars(); err != nil {
-		return err
-	}
-
-	if err := initRepoAvatars(); err != nil {
-		return err
-	}
-
-	if err := initLFS(); err != nil {
-		return err
-	}
-
-	if err := initRepoArchives(); err != nil {
-		return err
-	}
-
-	return initPackages()
+	return nil
 }
 
 // NewStorage takes a storage type and some config and returns an ObjectStorage or an error
@@ -201,3 +197,9 @@ func initPackages() (err error) {
 	Packages, err = NewStorage(setting.Packages.Storage.Type, &setting.Packages.Storage)
 	return err
 }
+
+func initBuilds() (err error) {
+	log.Info("Initialising Builds storage with type: %s", setting.Builds.Storage.Type)
+	Builds, err = NewStorage(setting.Builds.Storage.Type, &setting.Builds.Storage)
+	return err
+}