This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 9ae723f29 Fix/8787 cancel pipeline keeps running (#8832)
9ae723f29 is described below
commit 9ae723f29a988e0cc9e67b40e128e9b254d78049
Author: Daniele M. <[email protected]>
AuthorDate: Sat Apr 18 03:45:23 2026 +0100
Fix/8787 cancel pipeline keeps running (#8832)
* fix(pipeline): cancel pipeline returns 200 OK but keeps running (#8787)
Three independent bugs caused pipeline cancellation to silently fail:
1. CancelPipeline discarded errors from CancelTask and never cancelled
TASK_CREATED tasks in future stages. Now running tasks are cancelled
via context, non-running tasks are batch-updated to TASK_CANCELLED in
the DB, and errors are logged and returned.
2. gitextractor's storeRepoSnapshot (go-git path) had no ctx.Done()
checks in its commit/blame loops, making it unresponsive to
cancellation for 30+ minutes on large repos. Added cancellation
checkpoints following the pattern already used elsewhere in the file.
3. Cancelled tasks were marked TASK_FAILED instead of TASK_CANCELLED,
and ComputePipelineStatus never returned TASK_CANCELLED. Now RunTask
checks for context cancellation and writes TASK_CANCELLED, and
ComputePipelineStatus returns TASK_CANCELLED when the pipeline was
cancelled by the user.
Test gaps: RunTask deferred status logic, CancelPipeline flow, and
storeRepoSnapshot have no existing unit tests. These are pre-existing
gaps not introduced by this change. The only existing test,
TestComputePipelineStatus, has been extended to cover isCancelled=true.
Closes #8787
Related: #5585, #4188
* refactor(pipeline): improve CancelPipeline readability and correctness
Extract cancelRunningTasks and cancelPendingTasksInDB helpers from
CancelPipeline for better separation of concerns. Also fixes:
- .As(NotFound) replaced with .GetType() to prevent swallowing wrapped
errors
- Added TASK_RESUME to cancelPendingTasksInDB status filter
- Error message now includes pipeline ID for traceability
- Added TestCancelPipeline e2e tests with 3 subtests
---
backend/core/runner/run_task.go | 20 ++--
backend/plugins/gitextractor/parser/repo_gogit.go | 10 ++
backend/server/services/pipeline.go | 50 ++++++++-
backend/server/services/pipeline_runner.go | 10 +-
backend/test/e2e/services/pipeline_runner_test.go | 122 +++++++++++++++++++++-
5 files changed, 199 insertions(+), 13 deletions(-)
diff --git a/backend/core/runner/run_task.go b/backend/core/runner/run_task.go
index 8148a89de..2b848ea9b 100644
--- a/backend/core/runner/run_task.go
+++ b/backend/core/runner/run_task.go
@@ -77,7 +77,12 @@ func RunTask(
}
finishedAt := time.Now()
spentSeconds := finishedAt.Unix() - beganAt.Unix()
+ isCancelled := errors.Is(err, gocontext.Canceled) || ctx.Err()
== gocontext.Canceled
if err != nil {
+ taskStatus := models.TASK_FAILED
+ if isCancelled {
+ taskStatus = models.TASK_CANCELLED
+ }
lakeErr := errors.AsLakeErrorType(err)
subTaskName := "unknown"
if lakeErr = lakeErr.As(errors.SubtaskErr); lakeErr !=
nil {
@@ -87,16 +92,19 @@ func RunTask(
} else {
lakeErr = errors.Convert(err)
}
- dbe := db.UpdateColumns(task, []dal.DalSet{
- {ColumnName: "status", Value:
models.TASK_FAILED},
+ columns := []dal.DalSet{
+ {ColumnName: "status", Value: taskStatus},
{ColumnName: "message", Value: lakeErr.Error()},
{ColumnName: "error_name", Value:
lakeErr.Messages().Format()},
{ColumnName: "finished_at", Value: finishedAt},
{ColumnName: "spent_seconds", Value:
spentSeconds},
- {ColumnName: "failed_sub_task", Value:
subTaskName},
- })
+ }
+ if taskStatus == models.TASK_FAILED {
+ columns = append(columns,
dal.DalSet{ColumnName: "failed_sub_task", Value: subTaskName})
+ }
+ dbe := db.UpdateColumns(task, columns)
if dbe != nil {
- logger.Error(dbe, "failed to finalize task
status into db (task failed)")
+ logger.Error(dbe, "failed to finalize task
status into db (task %s)", taskStatus)
}
} else {
dbe := db.UpdateColumns(task, []dal.DalSet{
@@ -116,7 +124,7 @@ func RunTask(
dal.Where("id=?", task.PipelineId),
))
// not return err if the `SkipOnFail` is true and the error is
not canceled
- if dbPipeline.SkipOnFail && !errors.Is(err, gocontext.Canceled)
{
+ if dbPipeline.SkipOnFail && !isCancelled {
err = nil
}
}()
diff --git a/backend/plugins/gitextractor/parser/repo_gogit.go
b/backend/plugins/gitextractor/parser/repo_gogit.go
index 8837aa383..29deef315 100644
--- a/backend/plugins/gitextractor/parser/repo_gogit.go
+++ b/backend/plugins/gitextractor/parser/repo_gogit.go
@@ -524,6 +524,11 @@ func (r *GogitRepoCollector) storeRepoSnapshot(subtaskCtx
plugin.SubTaskContext,
ctx := subtaskCtx.GetContext()
snapshot := make(map[string][]string) // {"filePathAndName": ["line1
commit sha", "line2 commit sha"]}
for _, commit := range commitList {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ default:
+ }
commitTree, firstParentTree, err :=
r.getCurrentAndParentTree(ctx, commit)
if err != nil {
return err
@@ -533,6 +538,11 @@ func (r *GogitRepoCollector) storeRepoSnapshot(subtaskCtx
plugin.SubTaskContext,
return err
}
for _, p := range patch.Stats() {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ default:
+ }
fileName := p.Name
if _, ok := snapshot[fileName]; !ok {
snapshot[fileName] = []string{}
diff --git a/backend/server/services/pipeline.go
b/backend/server/services/pipeline.go
index 080028079..f998f3605 100644
--- a/backend/server/services/pipeline.go
+++ b/backend/server/services/pipeline.go
@@ -461,10 +461,54 @@ func CancelPipeline(pipelineId uint64) errors.Error {
if count == 0 {
return nil
}
- for _, pendingTask := range pendingTasks {
- _ = CancelTask(pendingTask.ID)
+ var runningTaskIds []uint64
+ var pendingTaskIds []uint64
+ for _, task := range pendingTasks {
+ if task.Status == models.TASK_RUNNING {
+ runningTaskIds = append(runningTaskIds, task.ID)
+ } else {
+ pendingTaskIds = append(pendingTaskIds, task.ID)
+ }
+ }
+ failedCancels := cancelRunningTasks(runningTaskIds) +
cancelPendingTasksInDB(pendingTaskIds)
+ if failedCancels > 0 {
+ return errors.Default.New(fmt.Sprintf("failed to cancel %d
task(s) for pipeline #%d", failedCancels, pipelineId))
+ }
+ return nil
+}
+
+// cancelRunningTasks cancels tasks that are actively running by triggering
+// their context cancellation. Returns the number of tasks that failed to
cancel.
+func cancelRunningTasks(taskIds []uint64) int {
+ failCount := 0
+ for _, taskId := range taskIds {
+ if err := CancelTask(taskId); err != nil {
+ if err.GetType() == errors.NotFound {
+ continue // task no longer tracked in-memory
(finished or context lost after restart)
+ }
+ globalPipelineLog.Error(err, "failed to cancel running
task #%d", taskId)
+ failCount++
+ }
+ }
+ return failCount
+}
+
+// cancelPendingTasksInDB marks non-running pending tasks as cancelled directly
+// in the database. Returns the number of tasks that failed to update.
+func cancelPendingTasksInDB(taskIds []uint64) int {
+ if len(taskIds) == 0 {
+ return 0
+ }
+ err := db.UpdateColumn(
+ &models.Task{},
+ "status", models.TASK_CANCELLED,
+ dal.Where("id IN ? AND status IN ?", taskIds,
[]string{models.TASK_CREATED, models.TASK_RERUN, models.TASK_RESUME}),
+ )
+ if err != nil {
+ globalPipelineLog.Error(err, "failed to cancel %d pending tasks
in DB", len(taskIds))
+ return len(taskIds)
}
- return errors.Convert(err)
+ return 0
}
// getPipelineLogsPath gets the logs directory of this pipeline
diff --git a/backend/server/services/pipeline_runner.go
b/backend/server/services/pipeline_runner.go
index 0f215a061..8f7fcd241 100644
--- a/backend/server/services/pipeline_runner.go
+++ b/backend/server/services/pipeline_runner.go
@@ -107,9 +107,10 @@ func runPipeline(pipelineId uint64) errors.Error {
}
// ComputePipelineStatus determines pipeline status by its latest(rerun
included) tasks statuses
-// 1. TASK_COMPLETED: all tasks were executed successfully
-// 2. TASK_FAILED: SkipOnFail=false with failed task(s)
-// 3. TASK_PARTIAL: SkipOnFail=true with failed task(s)
+// 1. TASK_CANCELLED: pipeline was cancelled by the user (takes priority)
+// 2. TASK_COMPLETED: all tasks were executed successfully
+// 3. TASK_FAILED: SkipOnFail=false with failed task(s)
+// 4. TASK_PARTIAL: SkipOnFail=true with failed task(s)
func ComputePipelineStatus(pipeline *models.Pipeline, isCancelled bool)
(string, errors.Error) {
tasks, err := GetLatestTasksOfPipeline(pipeline)
if err != nil {
@@ -134,6 +135,9 @@ func ComputePipelineStatus(pipeline *models.Pipeline,
isCancelled bool) (string,
return "", errors.Default.New("unexpected status, did you call
computePipelineStatus at a wrong timing?")
}
+ if isCancelled {
+ return models.TASK_CANCELLED, nil
+ }
if failed == 0 {
return models.TASK_COMPLETED, nil
}
diff --git a/backend/test/e2e/services/pipeline_runner_test.go
b/backend/test/e2e/services/pipeline_runner_test.go
index 0cb6b9142..e8a5f66ec 100644
--- a/backend/test/e2e/services/pipeline_runner_test.go
+++ b/backend/test/e2e/services/pipeline_runner_test.go
@@ -18,11 +18,14 @@ limitations under the License.
package services
import (
+ "testing"
+ "time"
+
+ "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/server/services"
"github.com/apache/incubator-devlake/test/helper"
"github.com/stretchr/testify/assert"
- "testing"
)
func TestComputePipelineStatus(t *testing.T) {
@@ -163,4 +166,121 @@ func TestComputePipelineStatus(t *testing.T) {
status, err = services.ComputePipelineStatus(pipeline, false)
assert.Nil(t, err)
assert.Equal(t, models.TASK_PARTIAL, status)
+
+ // pipeline.status == "cancelled" if the pipeline was cancelled by the
user
+ // regardless of individual task statuses
+ task_row1_col1_rerun.Status = models.TASK_COMPLETED
+ err = db.Update(task_row1_col1_rerun)
+ assert.Nil(t, err)
+ status, err = services.ComputePipelineStatus(pipeline, true)
+ assert.Nil(t, err)
+ assert.Equal(t, models.TASK_CANCELLED, status)
+
+ // pipeline.status == "cancelled" even when some tasks failed
+ task_row1_col1_rerun.Status = models.TASK_FAILED
+ err = db.Update(task_row1_col1_rerun)
+ assert.Nil(t, err)
+ status, err = services.ComputePipelineStatus(pipeline, true)
+ assert.Nil(t, err)
+ assert.Equal(t, models.TASK_CANCELLED, status)
+}
+
+func TestCancelPipeline(t *testing.T) {
+ client := helper.StartDevLakeServer(t, nil)
+ db := client.GetDal()
+
+ t.Run("cancels pending pipeline and all its tasks", func(t *testing.T) {
+ pipeline := &models.Pipeline{
+ TotalTasks: 2,
+ Status: models.TASK_CREATED,
+ }
+ err := db.Create(pipeline)
+ assert.Nil(t, err)
+ assert.NotZero(t, pipeline.ID)
+
+ task1 := &models.Task{
+ PipelineId: pipeline.ID,
+ PipelineRow: 1,
+ PipelineCol: 1,
+ Plugin: "github",
+ Status: models.TASK_CREATED,
+ }
+ task2 := &models.Task{
+ PipelineId: pipeline.ID,
+ PipelineRow: 1,
+ PipelineCol: 2,
+ Plugin: "gitextractor",
+ Status: models.TASK_CREATED,
+ }
+ err = db.Create(task1)
+ assert.Nil(t, err)
+ assert.NotZero(t, task1.ID)
+ err = db.Create(task2)
+ assert.Nil(t, err)
+ assert.NotZero(t, task2.ID)
+
+ err = services.CancelPipeline(pipeline.ID)
+ assert.Nil(t, err)
+
+ cancelledPipeline := &models.Pipeline{}
+ err = db.First(cancelledPipeline, dal.Where("id = ?",
pipeline.ID))
+ assert.Nil(t, err)
+ assert.Equal(t, models.TASK_CANCELLED, cancelledPipeline.Status)
+
+ cancelledTask1, err := services.GetTask(task1.ID)
+ assert.Nil(t, err)
+ assert.Equal(t, models.TASK_CANCELLED, cancelledTask1.Status)
+ cancelledTask2, err := services.GetTask(task2.ID)
+ assert.Nil(t, err)
+ assert.Equal(t, models.TASK_CANCELLED, cancelledTask2.Status)
+ })
+
+ t.Run("cancels pending tasks but leaves completed tasks unchanged",
func(t *testing.T) {
+ pipeline := &models.Pipeline{
+ TotalTasks: 2,
+ Status: models.TASK_RUNNING,
+ }
+ err := db.Create(pipeline)
+ assert.Nil(t, err)
+ assert.NotZero(t, pipeline.ID)
+
+ finishedAt := time.Now()
+ completedTask := &models.Task{
+ PipelineId: pipeline.ID,
+ PipelineRow: 1,
+ PipelineCol: 1,
+ Plugin: "github",
+ Status: models.TASK_COMPLETED,
+ FinishedAt: &finishedAt,
+ }
+ pendingTask := &models.Task{
+ PipelineId: pipeline.ID,
+ PipelineRow: 2,
+ PipelineCol: 1,
+ Plugin: "refdiff",
+ Status: models.TASK_CREATED,
+ }
+ err = db.Create(completedTask)
+ assert.Nil(t, err)
+ assert.NotZero(t, completedTask.ID)
+ err = db.Create(pendingTask)
+ assert.Nil(t, err)
+ assert.NotZero(t, pendingTask.ID)
+
+ err = services.CancelPipeline(pipeline.ID)
+ assert.Nil(t, err)
+
+ reloadedCompleted, err := services.GetTask(completedTask.ID)
+ assert.Nil(t, err)
+ assert.Equal(t, models.TASK_COMPLETED, reloadedCompleted.Status)
+
+ reloadedPending, err := services.GetTask(pendingTask.ID)
+ assert.Nil(t, err)
+ assert.Equal(t, models.TASK_CANCELLED, reloadedPending.Status)
+ })
+
+ t.Run("returns error for non-existent pipeline", func(t *testing.T) {
+ err := services.CancelPipeline(999999)
+ assert.NotNil(t, err)
+ })
}