mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2024-11-08 08:04:21 +00:00
Clear up old Actions logs (#31735)
Part of #24256. Clear up old action logs to free up storage space. Users will see a message indicating that the log has been cleared if they view old tasks. <img width="1361" alt="image" src="https://github.com/user-attachments/assets/9f0f3a3a-bc5a-402f-90ca-49282d196c22"> Docs: https://gitea.com/gitea/docs/pulls/40 --------- Co-authored-by: silverwind <me@silverwind.io> (cherry picked from commit 687c1182482ad9443a5911c068b317a91c91d586) Conflicts: custom/conf/app.example.ini routers/web/repo/actions/view.go trivial context conflict
This commit is contained in:
parent
5734499778
commit
0c40cff9a4
|
@ -2710,7 +2710,9 @@ LEVEL = Info
|
||||||
;ENABLED = true
|
;ENABLED = true
|
||||||
;; Default address to get action plugins, e.g. the default value means downloading from "https://code.forgejo.org/actions/checkout" for "uses: actions/checkout@v3"
|
;; Default address to get action plugins, e.g. the default value means downloading from "https://code.forgejo.org/actions/checkout" for "uses: actions/checkout@v3"
|
||||||
;DEFAULT_ACTIONS_URL = https://code.forgejo.org
|
;DEFAULT_ACTIONS_URL = https://code.forgejo.org
|
||||||
;; Default artifact retention time in days, default is 90 days
|
;; Logs retention time in days. Old logs will be deleted after this period.
|
||||||
|
;LOG_RETENTION_DAYS = 365
|
||||||
|
;; Default artifact retention time in days. Artifacts could have their own retention periods by setting the `retention-days` option in `actions/upload-artifact` step.
|
||||||
;ARTIFACT_RETENTION_DAYS = 90
|
;ARTIFACT_RETENTION_DAYS = 90
|
||||||
;; Timeout to stop the task which have running status, but haven't been updated for a long time
|
;; Timeout to stop the task which have running status, but haven't been updated for a long time
|
||||||
;ZOMBIE_TASK_TIMEOUT = 10m
|
;ZOMBIE_TASK_TIMEOUT = 10m
|
||||||
|
|
|
@ -35,7 +35,7 @@ type ActionTask struct {
|
||||||
RunnerID int64 `xorm:"index"`
|
RunnerID int64 `xorm:"index"`
|
||||||
Status Status `xorm:"index"`
|
Status Status `xorm:"index"`
|
||||||
Started timeutil.TimeStamp `xorm:"index"`
|
Started timeutil.TimeStamp `xorm:"index"`
|
||||||
Stopped timeutil.TimeStamp
|
Stopped timeutil.TimeStamp `xorm:"index(stopped_log_expired)"`
|
||||||
|
|
||||||
RepoID int64 `xorm:"index"`
|
RepoID int64 `xorm:"index"`
|
||||||
OwnerID int64 `xorm:"index"`
|
OwnerID int64 `xorm:"index"`
|
||||||
|
@ -51,8 +51,8 @@ type ActionTask struct {
|
||||||
LogInStorage bool // read log from database or from storage
|
LogInStorage bool // read log from database or from storage
|
||||||
LogLength int64 // lines count
|
LogLength int64 // lines count
|
||||||
LogSize int64 // blob size
|
LogSize int64 // blob size
|
||||||
LogIndexes LogIndexes `xorm:"LONGBLOB"` // line number to offset
|
LogIndexes LogIndexes `xorm:"LONGBLOB"` // line number to offset
|
||||||
LogExpired bool // files that are too old will be deleted
|
LogExpired bool `xorm:"index(stopped_log_expired)"` // files that are too old will be deleted
|
||||||
|
|
||||||
Created timeutil.TimeStamp `xorm:"created"`
|
Created timeutil.TimeStamp `xorm:"created"`
|
||||||
Updated timeutil.TimeStamp `xorm:"updated index"`
|
Updated timeutil.TimeStamp `xorm:"updated index"`
|
||||||
|
@ -470,6 +470,16 @@ func StopTask(ctx context.Context, taskID int64, status Status) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func FindOldTasksToExpire(ctx context.Context, olderThan timeutil.TimeStamp, limit int) ([]*ActionTask, error) {
|
||||||
|
e := db.GetEngine(ctx)
|
||||||
|
|
||||||
|
tasks := make([]*ActionTask, 0, limit)
|
||||||
|
// Check "stopped > 0" to avoid deleting tasks that are still running
|
||||||
|
return tasks, e.Where("stopped > 0 AND stopped < ? AND log_expired = ?", olderThan, false).
|
||||||
|
Limit(limit).
|
||||||
|
Find(&tasks)
|
||||||
|
}
|
||||||
|
|
||||||
func isSubset(set, subset []string) bool {
|
func isSubset(set, subset []string) bool {
|
||||||
m := make(container.Set[string], len(set))
|
m := make(container.Set[string], len(set))
|
||||||
for _, v := range set {
|
for _, v := range set {
|
||||||
|
|
|
@ -597,6 +597,8 @@ var migrations = []Migration{
|
||||||
NewMigration("Add force-push branch protection support", v1_23.AddForcePushBranchProtection),
|
NewMigration("Add force-push branch protection support", v1_23.AddForcePushBranchProtection),
|
||||||
// v301 -> v302
|
// v301 -> v302
|
||||||
NewMigration("Add skip_secondary_authorization option to oauth2 application table", v1_23.AddSkipSecondaryAuthColumnToOAuth2ApplicationTable),
|
NewMigration("Add skip_secondary_authorization option to oauth2 application table", v1_23.AddSkipSecondaryAuthColumnToOAuth2ApplicationTable),
|
||||||
|
// v302 -> v303
|
||||||
|
NewMigration("Add index to action_task stopped log_expired", v1_23.AddIndexToActionTaskStoppedLogExpired),
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetCurrentDBVersion returns the current db version
|
// GetCurrentDBVersion returns the current db version
|
||||||
|
|
18
models/migrations/v1_23/v302.go
Normal file
18
models/migrations/v1_23/v302.go
Normal file
|
@ -0,0 +1,18 @@
|
||||||
|
// Copyright 2024 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package v1_23 //nolint
|
||||||
|
|
||||||
|
import (
|
||||||
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
|
|
||||||
|
"xorm.io/xorm"
|
||||||
|
)
|
||||||
|
|
||||||
|
func AddIndexToActionTaskStoppedLogExpired(x *xorm.Engine) error {
|
||||||
|
type ActionTask struct {
|
||||||
|
Stopped timeutil.TimeStamp `xorm:"index(stopped_log_expired)"`
|
||||||
|
LogExpired bool `xorm:"index(stopped_log_expired)"`
|
||||||
|
}
|
||||||
|
return x.Sync(new(ActionTask))
|
||||||
|
}
|
|
@ -12,10 +12,11 @@ import (
|
||||||
// Actions settings
|
// Actions settings
|
||||||
var (
|
var (
|
||||||
Actions = struct {
|
Actions = struct {
|
||||||
LogStorage *Storage // how the created logs should be stored
|
|
||||||
ArtifactStorage *Storage // how the created artifacts should be stored
|
|
||||||
ArtifactRetentionDays int64 `ini:"ARTIFACT_RETENTION_DAYS"`
|
|
||||||
Enabled bool
|
Enabled bool
|
||||||
|
LogStorage *Storage // how the created logs should be stored
|
||||||
|
LogRetentionDays int64 `ini:"LOG_RETENTION_DAYS"`
|
||||||
|
ArtifactStorage *Storage // how the created artifacts should be stored
|
||||||
|
ArtifactRetentionDays int64 `ini:"ARTIFACT_RETENTION_DAYS"`
|
||||||
DefaultActionsURL defaultActionsURL `ini:"DEFAULT_ACTIONS_URL"`
|
DefaultActionsURL defaultActionsURL `ini:"DEFAULT_ACTIONS_URL"`
|
||||||
ZombieTaskTimeout time.Duration `ini:"ZOMBIE_TASK_TIMEOUT"`
|
ZombieTaskTimeout time.Duration `ini:"ZOMBIE_TASK_TIMEOUT"`
|
||||||
EndlessTaskTimeout time.Duration `ini:"ENDLESS_TASK_TIMEOUT"`
|
EndlessTaskTimeout time.Duration `ini:"ENDLESS_TASK_TIMEOUT"`
|
||||||
|
@ -61,10 +62,17 @@ func loadActionsFrom(rootCfg ConfigProvider) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// default to 1 year
|
||||||
|
if Actions.LogRetentionDays <= 0 {
|
||||||
|
Actions.LogRetentionDays = 365
|
||||||
|
}
|
||||||
|
|
||||||
actionsSec, _ := rootCfg.GetSection("actions.artifacts")
|
actionsSec, _ := rootCfg.GetSection("actions.artifacts")
|
||||||
|
|
||||||
Actions.ArtifactStorage, err = getStorage(rootCfg, "actions_artifacts", "", actionsSec)
|
Actions.ArtifactStorage, err = getStorage(rootCfg, "actions_artifacts", "", actionsSec)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
// default to 90 days in Github Actions
|
// default to 90 days in Github Actions
|
||||||
if Actions.ArtifactRetentionDays <= 0 {
|
if Actions.ArtifactRetentionDays <= 0 {
|
||||||
|
@ -75,5 +83,5 @@ func loadActionsFrom(rootCfg ConfigProvider) error {
|
||||||
Actions.EndlessTaskTimeout = sec.Key("ENDLESS_TASK_TIMEOUT").MustDuration(3 * time.Hour)
|
Actions.EndlessTaskTimeout = sec.Key("ENDLESS_TASK_TIMEOUT").MustDuration(3 * time.Hour)
|
||||||
Actions.AbandonedJobTimeout = sec.Key("ABANDONED_JOB_TIMEOUT").MustDuration(24 * time.Hour)
|
Actions.AbandonedJobTimeout = sec.Key("ABANDONED_JOB_TIMEOUT").MustDuration(24 * time.Hour)
|
||||||
|
|
||||||
return err
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -3827,6 +3827,7 @@ runs.no_workflows.quick_start = Don't know how to start with Forgejo Actions? Se
|
||||||
runs.no_workflows.documentation = For more information on Forgejo Actions, see <a target="_blank" rel="noopener noreferrer" href="%s">the documentation</a>.
|
runs.no_workflows.documentation = For more information on Forgejo Actions, see <a target="_blank" rel="noopener noreferrer" href="%s">the documentation</a>.
|
||||||
runs.no_runs = The workflow has no runs yet.
|
runs.no_runs = The workflow has no runs yet.
|
||||||
runs.empty_commit_message = (empty commit message)
|
runs.empty_commit_message = (empty commit message)
|
||||||
|
runs.expire_log_message = Logs have been purged because they were too old.
|
||||||
|
|
||||||
workflow.disable = Disable workflow
|
workflow.disable = Disable workflow
|
||||||
workflow.disable_success = Workflow "%s" disabled successfully.
|
workflow.disable_success = Workflow "%s" disabled successfully.
|
||||||
|
|
|
@ -271,6 +271,27 @@ func ViewPost(ctx *context_module.Context) {
|
||||||
|
|
||||||
step := steps[cursor.Step]
|
step := steps[cursor.Step]
|
||||||
|
|
||||||
|
// if task log is expired, return a consistent log line
|
||||||
|
if task.LogExpired {
|
||||||
|
if cursor.Cursor == 0 {
|
||||||
|
resp.Logs.StepsLog = append(resp.Logs.StepsLog, &ViewStepLog{
|
||||||
|
Step: cursor.Step,
|
||||||
|
Cursor: 1,
|
||||||
|
Lines: []*ViewStepLogLine{
|
||||||
|
{
|
||||||
|
Index: 1,
|
||||||
|
Message: ctx.Locale.TrString("actions.runs.expire_log_message"),
|
||||||
|
// Timestamp doesn't mean anything when the log is expired.
|
||||||
|
// Set it to the task's updated time since it's probably the time when the log has expired.
|
||||||
|
Timestamp: float64(task.Updated.AsTime().UnixNano()) / float64(time.Second),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Started: int64(step.Started),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
logLines := make([]*ViewStepLogLine, 0) // marshal to '[]' instead of 'null' in json
|
logLines := make([]*ViewStepLogLine, 0) // marshal to '[]' instead of 'null' in json
|
||||||
|
|
||||||
index := step.LogIndex + cursor.Cursor
|
index := step.LogIndex + cursor.Cursor
|
||||||
|
|
|
@ -5,18 +5,30 @@ package actions
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
"code.gitea.io/gitea/models/actions"
|
actions_model "code.gitea.io/gitea/models/actions"
|
||||||
|
actions_module "code.gitea.io/gitea/modules/actions"
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
|
"code.gitea.io/gitea/modules/setting"
|
||||||
"code.gitea.io/gitea/modules/storage"
|
"code.gitea.io/gitea/modules/storage"
|
||||||
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Cleanup removes expired actions logs, data and artifacts
|
// Cleanup removes expired actions logs, data and artifacts
|
||||||
func Cleanup(taskCtx context.Context) error {
|
func Cleanup(ctx context.Context) error {
|
||||||
// TODO: clean up expired actions logs
|
|
||||||
|
|
||||||
// clean up expired artifacts
|
// clean up expired artifacts
|
||||||
return CleanupArtifacts(taskCtx)
|
if err := CleanupArtifacts(ctx); err != nil {
|
||||||
|
return fmt.Errorf("cleanup artifacts: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// clean up old logs
|
||||||
|
if err := CleanupLogs(ctx); err != nil {
|
||||||
|
return fmt.Errorf("cleanup logs: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// CleanupArtifacts removes expired add need-deleted artifacts and set records expired status
|
// CleanupArtifacts removes expired add need-deleted artifacts and set records expired status
|
||||||
|
@ -28,13 +40,13 @@ func CleanupArtifacts(taskCtx context.Context) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func cleanExpiredArtifacts(taskCtx context.Context) error {
|
func cleanExpiredArtifacts(taskCtx context.Context) error {
|
||||||
artifacts, err := actions.ListNeedExpiredArtifacts(taskCtx)
|
artifacts, err := actions_model.ListNeedExpiredArtifacts(taskCtx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
log.Info("Found %d expired artifacts", len(artifacts))
|
log.Info("Found %d expired artifacts", len(artifacts))
|
||||||
for _, artifact := range artifacts {
|
for _, artifact := range artifacts {
|
||||||
if err := actions.SetArtifactExpired(taskCtx, artifact.ID); err != nil {
|
if err := actions_model.SetArtifactExpired(taskCtx, artifact.ID); err != nil {
|
||||||
log.Error("Cannot set artifact %d expired: %v", artifact.ID, err)
|
log.Error("Cannot set artifact %d expired: %v", artifact.ID, err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
@ -52,13 +64,13 @@ const deleteArtifactBatchSize = 100
|
||||||
|
|
||||||
func cleanNeedDeleteArtifacts(taskCtx context.Context) error {
|
func cleanNeedDeleteArtifacts(taskCtx context.Context) error {
|
||||||
for {
|
for {
|
||||||
artifacts, err := actions.ListPendingDeleteArtifacts(taskCtx, deleteArtifactBatchSize)
|
artifacts, err := actions_model.ListPendingDeleteArtifacts(taskCtx, deleteArtifactBatchSize)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
log.Info("Found %d artifacts pending deletion", len(artifacts))
|
log.Info("Found %d artifacts pending deletion", len(artifacts))
|
||||||
for _, artifact := range artifacts {
|
for _, artifact := range artifacts {
|
||||||
if err := actions.SetArtifactDeleted(taskCtx, artifact.ID); err != nil {
|
if err := actions_model.SetArtifactDeleted(taskCtx, artifact.ID); err != nil {
|
||||||
log.Error("Cannot set artifact %d deleted: %v", artifact.ID, err)
|
log.Error("Cannot set artifact %d deleted: %v", artifact.ID, err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
@ -75,3 +87,40 @@ func cleanNeedDeleteArtifacts(taskCtx context.Context) error {
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const deleteLogBatchSize = 100
|
||||||
|
|
||||||
|
// CleanupLogs removes logs which are older than the configured retention time
|
||||||
|
func CleanupLogs(ctx context.Context) error {
|
||||||
|
olderThan := timeutil.TimeStampNow().AddDuration(-time.Duration(setting.Actions.LogRetentionDays) * 24 * time.Hour)
|
||||||
|
|
||||||
|
count := 0
|
||||||
|
for {
|
||||||
|
tasks, err := actions_model.FindOldTasksToExpire(ctx, olderThan, deleteLogBatchSize)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("find old tasks: %w", err)
|
||||||
|
}
|
||||||
|
for _, task := range tasks {
|
||||||
|
if err := actions_module.RemoveLogs(ctx, task.LogInStorage, task.LogFilename); err != nil {
|
||||||
|
log.Error("Failed to remove log %s (in storage %v) of task %v: %v", task.LogFilename, task.LogInStorage, task.ID, err)
|
||||||
|
// do not return error here, continue to next task
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
task.LogIndexes = nil // clear log indexes since it's a heavy field
|
||||||
|
task.LogExpired = true
|
||||||
|
if err := actions_model.UpdateTask(ctx, task, "log_indexes", "log_expired"); err != nil {
|
||||||
|
log.Error("Failed to update task %v: %v", task.ID, err)
|
||||||
|
// do not return error here, continue to next task
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
count++
|
||||||
|
log.Trace("Removed log %s of task %v", task.LogFilename, task.ID)
|
||||||
|
}
|
||||||
|
if len(tasks) < deleteLogBatchSize {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info("Removed %d logs", count)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
|
@ -68,7 +68,7 @@ func registerScheduleTasks() {
|
||||||
func registerActionsCleanup() {
|
func registerActionsCleanup() {
|
||||||
RegisterTaskFatal("cleanup_actions", &BaseConfig{
|
RegisterTaskFatal("cleanup_actions", &BaseConfig{
|
||||||
Enabled: true,
|
Enabled: true,
|
||||||
RunAtStart: true,
|
RunAtStart: false,
|
||||||
Schedule: "@midnight",
|
Schedule: "@midnight",
|
||||||
}, func(ctx context.Context, _ *user_model.User, _ Config) error {
|
}, func(ctx context.Context, _ *user_model.User, _ Config) error {
|
||||||
return actions_service.Cleanup(ctx)
|
return actions_service.Cleanup(ctx)
|
||||||
|
|
Loading…
Reference in a new issue