ddmt/gitdts
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: use dbfs to store log

Browse Source
This commit is contained in:
Jason Song
2022-10-18 13:32:52 +08:00
parent f633ec9704
commit d7b99e0028
7 changed files with 206 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
routers/api/bots/runner/runner.go
View File

@@ -12,9 +12,9 @@ import (
"code.gitea.io/gitea/core"
bots_model "code.gitea.io/gitea/models/bots"
"code.gitea.io/gitea/modules/bots"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/timeutil"
runnerv1 "gitea.com/gitea/proto-go/runner/v1"
"gitea.com/gitea/proto-go/runner/v1/runnerv1connect"
@@ -149,7 +149,7 @@ func (s *Service) UpdateTask(
) (*connect.Response[runnerv1.UpdateTaskResponse], error) {
res := connect.NewResponse(&runnerv1.UpdateTaskResponse{})
if err := bots_model.UpdateTask(req.Msg.State); err != nil {
if err := bots_model.UpdateTaskByState(req.Msg.State); err != nil {
return nil, status.Errorf(codes.Internal, "update task: %v", err)
}
@@ -163,27 +163,38 @@ func (s *Service) UpdateLog(
) (*connect.Response[runnerv1.UpdateLogResponse], error) {
res := connect.NewResponse(&runnerv1.UpdateLogResponse{})
if len(req.Msg.Rows) == 0 {
// TODO: should be 1 + the max id of stored log
res.Msg.AckIndex = req.Msg.Index
task, err := bots_model.GetTaskByID(ctx, req.Msg.TaskId)
if err != nil {
return nil, status.Errorf(codes.Internal, "get task: %v", err)
}
if task.LogURL == "" {
task.LogURL = fmt.Sprintf("dbfs:///bots/tasks/%d.log", task.ID)
if err := bots_model.UpdateTask(ctx, task, "log_url"); err != nil {
return nil, status.Errorf(codes.Internal, "update task: %v", err)
}
}
ack := task.LogLength
if len(req.Msg.Rows) == 0 || req.Msg.Index > ack || int64(len(req.Msg.Rows))+req.Msg.Index <= ack {
res.Msg.AckIndex = ack
return res, nil
}
rowIndex := req.Msg.Index
rows := make([]*bots_model.TaskLog, len(req.Msg.Rows))
for i, v := range req.Msg.Rows {
rows[i] = &bots_model.TaskLog{
ID: rowIndex + int64(i),
Timestamp: timeutil.TimeStamp(v.Time.AsTime().Unix()),
Content: v.Content,
}
rows := req.Msg.Rows[ack-req.Msg.Index:]
ns, err := bots.WriteLogs(ctx, task.LogURL, task.LogSize, rows)
if err != nil {
return nil, status.Errorf(codes.Internal, "write logs: %v", err)
}
task.LogLength += int64(len(rows))
for _, n := range ns {
task.LogIndexes = append(task.LogIndexes, task.LogSize)
task.LogSize += int64(n)
}
if err := bots_model.UpdateTask(ctx, task, "log_indexes", "log_length", "log_size"); err != nil {
return nil, status.Errorf(codes.Internal, "update task: %v", err)
}
ack, err := bots_model.InsertTaskLogs(req.Msg.TaskId, rows)
if err != nil {
return nil, status.Errorf(codes.Internal, "insert task log: %v", err)
}
res.Msg.AckIndex = ack
res.Msg.AckIndex = task.LogLength
if req.Msg.NoMore {
// TODO: transfer logs to storage from db

16
routers/web/dev/buildview.go
View File

@@ -3,11 +3,14 @@ package dev
import (
"fmt"
"net/http"
"time"
"code.gitea.io/gitea/core"
bots_model "code.gitea.io/gitea/models/bots"
"code.gitea.io/gitea/modules/bots"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/web"
runnerv1 "gitea.com/gitea/proto-go/runner/v1"
)
func BuildView(ctx *context.Context) {
@@ -78,7 +81,7 @@ type BuildViewStepLog struct {
}
type BuildViewStepLogLine struct {
Ln int `json:"ln"`
Ln int64 `json:"ln"`
M string `json:"m"`
T float64 `json:"t"`
}
@@ -172,9 +175,12 @@ func BuildViewPost(ctx *context.Context) {
for _, cursor := range req.StepLogCursors {
if cursor.Expanded {
step := steps[cursor.StepIndex]
var logRows []*bots_model.TaskLog
var logRows []*runnerv1.LogRow
if cursor.Cursor < step.LogLength || step.LogLength < 0 {
logRows, err = bots_model.GetTaskLogs(task.ID, step.LogIndex+cursor.Cursor, step.LogLength-cursor.Cursor)
index := step.LogIndex + cursor.Cursor
length := step.LogLength - cursor.Cursor
offset := task.LogIndexes[index]
logRows, err = bots.ReadLogs(ctx, task.LogURL, offset, length)
if err != nil {
ctx.Error(http.StatusInternalServerError, err.Error())
return
@@ -183,9 +189,9 @@ func BuildViewPost(ctx *context.Context) {
logLines := make([]BuildViewStepLogLine, len(logRows))
for i, row := range logRows {
logLines[i] = BuildViewStepLogLine{
Ln: i,
Ln: cursor.Cursor + int64(i),
M: row.Content,
T: float64(row.Timestamp),
T: float64(row.Time.AsTime().UnixNano()) / float64(time.Second),
}
}
resp.LogsData.StreamingLogs = append(resp.LogsData.StreamingLogs, BuildViewStepLog{