From c7420e5d3f66329a4f873d294c8689751563e70c Mon Sep 17 00:00:00 2001 From: Zettat123 Date: Mon, 3 Nov 2025 19:32:26 -0700 Subject: [PATCH 1/4] Add a doctor command to fix inconsistent run status (#35840) incorrectly marked as `StatusWaiting` even though all the jobs are in done status. These runs cannot be run or cancelled. This PR adds a new doctor command to fix the inconsistent run status. ``` gitea doctor check --run fix-actions-unfinished-run-status --fix ``` Thanks to @ChristopherHX for the test. --- models/actions/run_test.go | 2 +- models/fixtures/action_run.yml | 40 ++++++++++++ models/fixtures/action_run_job.yml | 28 ++++++++ models/fixtures/action_task.yml | 39 +++++++++++ models/fixtures/repo_unit.yml | 14 ++++ services/doctor/actions.go | 101 +++++++++++++++++++++++++++++ services/doctor/actions_test.go | 24 +++++++ 7 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 services/doctor/actions_test.go diff --git a/models/actions/run_test.go b/models/actions/run_test.go index c65ee1521d995..6463d6f721211 100644 --- a/models/actions/run_test.go +++ b/models/actions/run_test.go @@ -30,6 +30,6 @@ func TestUpdateRepoRunsNumbers(t *testing.T) { err = updateRepoRunsNumbers(t.Context(), repo) assert.NoError(t, err) repo = unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 4}) - assert.Equal(t, 4, repo.NumActionRuns) + assert.Equal(t, 5, repo.NumActionRuns) assert.Equal(t, 3, repo.NumClosedActionRuns) } diff --git a/models/fixtures/action_run.yml b/models/fixtures/action_run.yml index 09dfa6cccbba3..44b131c961272 100644 --- a/models/fixtures/action_run.yml +++ b/models/fixtures/action_run.yml @@ -139,3 +139,43 @@ updated: 1683636626 need_approval: 0 approved_by: 0 +- + id: 804 + title: "use a private action" + repo_id: 60 + owner_id: 40 + workflow_id: "run.yaml" + index: 189 + trigger_user_id: 40 + ref: "refs/heads/master" + commit_sha: "6e64b26de7ba966d01d90ecfaf5c7f14ef203e86" + event: "push" + trigger_event: "push" + is_fork_pull_request: 0 + status: 1 + started: 1683636528 + stopped: 1683636626 + created: 1683636108 + updated: 1683636626 + need_approval: 0 + approved_by: 0 +- + id: 805 + title: "update actions" + repo_id: 4 + owner_id: 1 + workflow_id: "artifact.yaml" + index: 191 + trigger_user_id: 1 + ref: "refs/heads/master" + commit_sha: "c2d72f548424103f01ee1dc02889c1e2bff816b0" + event: "push" + trigger_event: "push" + is_fork_pull_request: 0 + status: 5 + started: 1683636528 + stopped: 1683636626 + created: 1683636108 + updated: 1683636626 + need_approval: 0 + approved_by: 0 diff --git a/models/fixtures/action_run_job.yml b/models/fixtures/action_run_job.yml index 6c06d94aa44ca..c5aeb4931cf17 100644 --- a/models/fixtures/action_run_job.yml +++ b/models/fixtures/action_run_job.yml @@ -129,3 +129,31 @@ status: 5 started: 1683636528 stopped: 1683636626 +- + id: 205 + run_id: 804 + repo_id: 6 + owner_id: 10 + commit_sha: 6e64b26de7ba966d01d90ecfaf5c7f14ef203e86 + is_fork_pull_request: 0 + name: job_2 + attempt: 1 + job_id: job_2 + task_id: 48 + status: 1 + started: 1683636528 + stopped: 1683636626 +- + id: 206 + run_id: 805 + repo_id: 4 + owner_id: 1 + commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0 + is_fork_pull_request: 0 + name: job_2 + attempt: 1 + job_id: job_2 + task_id: 56 + status: 3 + started: 1683636528 + stopped: 1683636626 diff --git a/models/fixtures/action_task.yml b/models/fixtures/action_task.yml index c79fb070506dd..a28ddd0add797 100644 --- a/models/fixtures/action_task.yml +++ b/models/fixtures/action_task.yml @@ -177,3 +177,42 @@ log_length: 0 log_size: 0 log_expired: 0 +- + id: 55 + job_id: 205 + attempt: 1 + runner_id: 1 + status: 6 # 6 is the status code for "running" + started: 1683636528 + stopped: 1683636626 + repo_id: 6 + owner_id: 10 + commit_sha: 6e64b26de7ba966d01d90ecfaf5c7f14ef203e86 + is_fork_pull_request: 0 + token_hash: b8d3962425466b6709b9ac51446f93260c54afe8e7b6d3686e34f991fb8a8953822b0deed86fe41a103f34bc48dbc478422b + token_salt: ERxJGHvg3I + token_last_eight: 182199eb + log_filename: collaborative-owner-test/1a/49.log + log_in_storage: 1 + log_length: 707 + log_size: 90179 + log_expired: 0 +- + id: 56 + attempt: 1 + runner_id: 1 + status: 3 # 3 is the status code for "cancelled" + started: 1683636528 + stopped: 1683636626 + repo_id: 4 + owner_id: 1 + commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0 + is_fork_pull_request: 0 + token_hash: 6d8ef48297195edcc8e22c70b3020eaa06c52976db67d39b4240c64a69a2cc1508825121b7b8394e48e00b1bf3718b2aaaab + token_salt: eeeeeeee + token_last_eight: eeeeeeee + log_filename: artifact-test2/2f/47.log + log_in_storage: 1 + log_length: 707 + log_size: 90179 + log_expired: 0 diff --git a/models/fixtures/repo_unit.yml b/models/fixtures/repo_unit.yml index f6b6252da1f88..4c3e37500f094 100644 --- a/models/fixtures/repo_unit.yml +++ b/models/fixtures/repo_unit.yml @@ -733,3 +733,17 @@ type: 3 config: "{\"IgnoreWhitespaceConflicts\":false,\"AllowMerge\":true,\"AllowRebase\":true,\"AllowRebaseMerge\":true,\"AllowSquash\":true}" created_unix: 946684810 + +- + id: 111 + repo_id: 3 + type: 10 + config: "{}" + created_unix: 946684810 + +- + id: 112 + repo_id: 4 + type: 10 + config: "{}" + created_unix: 946684810 diff --git a/services/doctor/actions.go b/services/doctor/actions.go index 28e26c88ebe70..cd3d19b724f8a 100644 --- a/services/doctor/actions.go +++ b/services/doctor/actions.go @@ -7,12 +7,17 @@ import ( "context" "fmt" + actions_model "code.gitea.io/gitea/models/actions" "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" unit_model "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/optional" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" repo_service "code.gitea.io/gitea/services/repository" + + "xorm.io/builder" ) func disableMirrorActionsUnit(ctx context.Context, logger log.Logger, autofix bool) error { @@ -59,6 +64,95 @@ func disableMirrorActionsUnit(ctx context.Context, logger log.Logger, autofix bo return nil } +func fixUnfinishedRunStatus(ctx context.Context, logger log.Logger, autofix bool) error { + total := 0 + inconsistent := 0 + fixed := 0 + + cond := builder.In("status", []actions_model.Status{ + actions_model.StatusWaiting, + actions_model.StatusRunning, + actions_model.StatusBlocked, + }).And(builder.Lt{"updated": timeutil.TimeStampNow().AddDuration(-setting.Actions.ZombieTaskTimeout)}) + + err := db.Iterate( + ctx, + cond, + func(ctx context.Context, run *actions_model.ActionRun) error { + total++ + + jobs, err := actions_model.GetRunJobsByRunID(ctx, run.ID) + if err != nil { + return fmt.Errorf("GetRunJobsByRunID: %w", err) + } + expected := actions_model.AggregateJobStatus(jobs) + if expected == run.Status { + return nil + } + + inconsistent++ + logger.Warn("Run %d (repo_id=%d, index=%d) has status %s, expected %s", run.ID, run.RepoID, run.Index, run.Status, expected) + + if !autofix { + return nil + } + + run.Started, run.Stopped = getRunTimestampsFromJobs(run, expected, jobs) + run.Status = expected + + if err := actions_model.UpdateRun(ctx, run, "status", "started", "stopped"); err != nil { + return fmt.Errorf("UpdateRun: %w", err) + } + fixed++ + + return nil + }, + ) + if err != nil { + logger.Critical("Unable to iterate unfinished runs: %v", err) + return err + } + + if inconsistent == 0 { + logger.Info("Checked %d unfinished runs; all statuses are consistent.", total) + return nil + } + + if autofix { + logger.Info("Checked %d unfinished runs; fixed %d of %d runs.", total, fixed, inconsistent) + } else { + logger.Warn("Checked %d unfinished runs; found %d runs need to be fixed", total, inconsistent) + } + + return nil +} + +func getRunTimestampsFromJobs(run *actions_model.ActionRun, newStatus actions_model.Status, jobs actions_model.ActionJobList) (started, stopped timeutil.TimeStamp) { + started = run.Started + if (newStatus.IsRunning() || newStatus.IsDone()) && started.IsZero() { + var earliest timeutil.TimeStamp + for _, job := range jobs { + if job.Started > 0 && (earliest.IsZero() || job.Started < earliest) { + earliest = job.Started + } + } + started = earliest + } + + stopped = run.Stopped + if newStatus.IsDone() && stopped.IsZero() { + var latest timeutil.TimeStamp + for _, job := range jobs { + if job.Stopped > latest { + latest = job.Stopped + } + } + stopped = latest + } + + return started, stopped +} + func init() { Register(&Check{ Title: "Disable the actions unit for all mirrors", @@ -67,4 +161,11 @@ func init() { Run: disableMirrorActionsUnit, Priority: 9, }) + Register(&Check{ + Title: "Fix inconsistent status for unfinished actions runs", + Name: "fix-actions-unfinished-run-status", + IsDefault: false, + Run: fixUnfinishedRunStatus, + Priority: 9, + }) } diff --git a/services/doctor/actions_test.go b/services/doctor/actions_test.go new file mode 100644 index 0000000000000..b2fd3d0d55362 --- /dev/null +++ b/services/doctor/actions_test.go @@ -0,0 +1,24 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package doctor + +import ( + "testing" + + actions_model "code.gitea.io/gitea/models/actions" + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/log" + + "github.com/stretchr/testify/assert" +) + +func Test_fixUnfinishedRunStatus(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + fixUnfinishedRunStatus(t.Context(), log.GetLogger(log.DEFAULT), true) + + // check if the run is cancelled by id + run := unittest.AssertExistsAndLoadBean(t, &actions_model.ActionRun{ID: 805}) + assert.Equal(t, actions_model.StatusCancelled, run.Status) +} From 64ca2ba37cfbdaa8ec11a0342c32cf9b2018dad1 Mon Sep 17 00:00:00 2001 From: Zettat123 Date: Mon, 3 Nov 2025 21:11:37 -0700 Subject: [PATCH 2/4] remove fixtures --- models/fixtures/action_run.yml | 21 +-------------------- models/fixtures/action_run_job.yml | 15 +-------------- models/fixtures/action_task.yml | 21 +-------------------- models/fixtures/repo_unit.yml | 7 ------- 4 files changed, 3 insertions(+), 61 deletions(-) diff --git a/models/fixtures/action_run.yml b/models/fixtures/action_run.yml index 44b131c961272..ac5e8303c35ab 100644 --- a/models/fixtures/action_run.yml +++ b/models/fixtures/action_run.yml @@ -139,26 +139,7 @@ updated: 1683636626 need_approval: 0 approved_by: 0 -- - id: 804 - title: "use a private action" - repo_id: 60 - owner_id: 40 - workflow_id: "run.yaml" - index: 189 - trigger_user_id: 40 - ref: "refs/heads/master" - commit_sha: "6e64b26de7ba966d01d90ecfaf5c7f14ef203e86" - event: "push" - trigger_event: "push" - is_fork_pull_request: 0 - status: 1 - started: 1683636528 - stopped: 1683636626 - created: 1683636108 - updated: 1683636626 - need_approval: 0 - approved_by: 0 + - id: 805 title: "update actions" diff --git a/models/fixtures/action_run_job.yml b/models/fixtures/action_run_job.yml index c5aeb4931cf17..04799b73ca064 100644 --- a/models/fixtures/action_run_job.yml +++ b/models/fixtures/action_run_job.yml @@ -129,20 +129,7 @@ status: 5 started: 1683636528 stopped: 1683636626 -- - id: 205 - run_id: 804 - repo_id: 6 - owner_id: 10 - commit_sha: 6e64b26de7ba966d01d90ecfaf5c7f14ef203e86 - is_fork_pull_request: 0 - name: job_2 - attempt: 1 - job_id: job_2 - task_id: 48 - status: 1 - started: 1683636528 - stopped: 1683636626 + - id: 206 run_id: 805 diff --git a/models/fixtures/action_task.yml b/models/fixtures/action_task.yml index a28ddd0add797..e1bc588dc59e1 100644 --- a/models/fixtures/action_task.yml +++ b/models/fixtures/action_task.yml @@ -177,26 +177,7 @@ log_length: 0 log_size: 0 log_expired: 0 -- - id: 55 - job_id: 205 - attempt: 1 - runner_id: 1 - status: 6 # 6 is the status code for "running" - started: 1683636528 - stopped: 1683636626 - repo_id: 6 - owner_id: 10 - commit_sha: 6e64b26de7ba966d01d90ecfaf5c7f14ef203e86 - is_fork_pull_request: 0 - token_hash: b8d3962425466b6709b9ac51446f93260c54afe8e7b6d3686e34f991fb8a8953822b0deed86fe41a103f34bc48dbc478422b - token_salt: ERxJGHvg3I - token_last_eight: 182199eb - log_filename: collaborative-owner-test/1a/49.log - log_in_storage: 1 - log_length: 707 - log_size: 90179 - log_expired: 0 + - id: 56 attempt: 1 diff --git a/models/fixtures/repo_unit.yml b/models/fixtures/repo_unit.yml index 4c3e37500f094..abdc8c746f653 100644 --- a/models/fixtures/repo_unit.yml +++ b/models/fixtures/repo_unit.yml @@ -734,13 +734,6 @@ config: "{\"IgnoreWhitespaceConflicts\":false,\"AllowMerge\":true,\"AllowRebase\":true,\"AllowRebaseMerge\":true,\"AllowSquash\":true}" created_unix: 946684810 -- - id: 111 - repo_id: 3 - type: 10 - config: "{}" - created_unix: 946684810 - - id: 112 repo_id: 4 From 0da25e4ad7cf0e0df048b0677a71b43f08c3140b Mon Sep 17 00:00:00 2001 From: Zettat123 Date: Mon, 3 Nov 2025 22:03:14 -0700 Subject: [PATCH 3/4] fix fixtures --- models/fixtures/action_run.yml | 2 +- models/fixtures/action_run_job.yml | 4 ++-- models/fixtures/action_task.yml | 3 ++- models/fixtures/repo_unit.yml | 2 +- services/doctor/actions_test.go | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/models/fixtures/action_run.yml b/models/fixtures/action_run.yml index ac5e8303c35ab..8018ae770fc22 100644 --- a/models/fixtures/action_run.yml +++ b/models/fixtures/action_run.yml @@ -141,7 +141,7 @@ approved_by: 0 - - id: 805 + id: 796 title: "update actions" repo_id: 4 owner_id: 1 diff --git a/models/fixtures/action_run_job.yml b/models/fixtures/action_run_job.yml index 04799b73ca064..73a81f34350be 100644 --- a/models/fixtures/action_run_job.yml +++ b/models/fixtures/action_run_job.yml @@ -131,8 +131,8 @@ stopped: 1683636626 - - id: 206 - run_id: 805 + id: 205 + run_id: 796 repo_id: 4 owner_id: 1 commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0 diff --git a/models/fixtures/action_task.yml b/models/fixtures/action_task.yml index e1bc588dc59e1..95d3ee25d18a4 100644 --- a/models/fixtures/action_task.yml +++ b/models/fixtures/action_task.yml @@ -179,7 +179,8 @@ log_expired: 0 - - id: 56 + id: 55 + job_id: 205 attempt: 1 runner_id: 1 status: 3 # 3 is the status code for "cancelled" diff --git a/models/fixtures/repo_unit.yml b/models/fixtures/repo_unit.yml index abdc8c746f653..fa89e3f12ad76 100644 --- a/models/fixtures/repo_unit.yml +++ b/models/fixtures/repo_unit.yml @@ -735,7 +735,7 @@ created_unix: 946684810 - - id: 112 + id: 111 repo_id: 4 type: 10 config: "{}" diff --git a/services/doctor/actions_test.go b/services/doctor/actions_test.go index b2fd3d0d55362..a3cca54b9fc91 100644 --- a/services/doctor/actions_test.go +++ b/services/doctor/actions_test.go @@ -19,6 +19,6 @@ func Test_fixUnfinishedRunStatus(t *testing.T) { fixUnfinishedRunStatus(t.Context(), log.GetLogger(log.DEFAULT), true) // check if the run is cancelled by id - run := unittest.AssertExistsAndLoadBean(t, &actions_model.ActionRun{ID: 805}) + run := unittest.AssertExistsAndLoadBean(t, &actions_model.ActionRun{ID: 796}) assert.Equal(t, actions_model.StatusCancelled, run.Status) } From 453fcd1328a308a7aa8bd7cfeeb97cbc931a41e7 Mon Sep 17 00:00:00 2001 From: Zettat123 Date: Tue, 4 Nov 2025 08:35:11 -0700 Subject: [PATCH 4/4] fix test --- models/fixtures/action_run_job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/fixtures/action_run_job.yml b/models/fixtures/action_run_job.yml index 73a81f34350be..789eb248a5c33 100644 --- a/models/fixtures/action_run_job.yml +++ b/models/fixtures/action_run_job.yml @@ -140,7 +140,7 @@ name: job_2 attempt: 1 job_id: job_2 - task_id: 56 + task_id: 55 status: 3 started: 1683636528 stopped: 1683636626