This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 092b8953c fix(gitlab): add missing commit_updated_at column used by MR
incremental filter (#9046)
092b8953c is described below
commit 092b8953ce3fdbaace1215ba01dc5ee450eff9a7
Author: Dirk <[email protected]>
AuthorDate: Fri Aug 14 02:40:19 2026 +0200
fix(gitlab): add missing commit_updated_at column used by MR incremental
filter (#9046)
#8959 changed `GetMergeRequestsIterator` to filter merge requests by
`GREATEST(gmr.gitlab_updated_at, COALESCE(gmr.commit_updated_at,
gmr.gitlab_updated_at))`,
but `commit_updated_at` does not exist on `_tool_gitlab_merge_requests`:
there is neither a model field nor a migration script, and nothing ever
writes the value.
As a result every incremental "Collect MR Notes" run (the only caller that
passes a stateful collector; the MR commit collector passes nil) aborts
with:
Error 1054 (42S22): Unknown column 'gmr.commit_updated_at' in 'where
clause'
This makes the intended behaviour actually work instead of reverting it:
* add `CommitUpdatedAt` to `GitlabMergeRequest` plus a migration script
* maintain the column in the MR commit extractor, setting it to the latest
authored date of the MR's commits, so MRs that received new commits
without
their own `updated_at` being bumped (e.g. force-pushes) are picked up
again
* the collector filter is unchanged and now resolves against a real column
Verified on MySQL 8 and PostgreSQL 14 (extractor UPDATE and the GREATEST
filter produce identical results on both).
---
.../20260812_add_mr_commit_updated_at.go | 50 ++++++++++++++++++++++
.../gitlab/models/migrationscripts/register.go | 1 +
backend/plugins/gitlab/models/mr.go | 5 +++
.../plugins/gitlab/tasks/mr_commit_extractor.go | 32 +++++++++++++-
backend/plugins/gitlab/tasks/shared.go | 3 +-
5 files changed, 89 insertions(+), 2 deletions(-)
diff --git
a/backend/plugins/gitlab/models/migrationscripts/20260812_add_mr_commit_updated_at.go
b/backend/plugins/gitlab/models/migrationscripts/20260812_add_mr_commit_updated_at.go
new file mode 100644
index 000000000..b88b66546
--- /dev/null
+++
b/backend/plugins/gitlab/models/migrationscripts/20260812_add_mr_commit_updated_at.go
@@ -0,0 +1,50 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*addMrCommitUpdatedAt)(nil)
+
+type mrCommitUpdatedAt struct {
+ CommitUpdatedAt *time.Time
+}
+
+func (mrCommitUpdatedAt) TableName() string {
+ return "_tool_gitlab_merge_requests"
+}
+
+type addMrCommitUpdatedAt struct{}
+
+func (*addMrCommitUpdatedAt) Up(basicRes context.BasicRes) errors.Error {
+ return
errors.Convert(basicRes.GetDal().AutoMigrate(&mrCommitUpdatedAt{}))
+}
+
+func (*addMrCommitUpdatedAt) Version() uint64 {
+ return 20260812000001
+}
+
+func (*addMrCommitUpdatedAt) Name() string {
+ return "gitlab: add commit_updated_at to merge requests"
+}
diff --git a/backend/plugins/gitlab/models/migrationscripts/register.go
b/backend/plugins/gitlab/models/migrationscripts/register.go
index e684c9530..65fa6f41b 100644
--- a/backend/plugins/gitlab/models/migrationscripts/register.go
+++ b/backend/plugins/gitlab/models/migrationscripts/register.go
@@ -54,5 +54,6 @@ func All() []plugin.MigrationScript {
new(addIsChildToPipelines240906),
new(addPrSizeExcludedFileExtensions),
new(addAdditionsDeletionsToMr),
+ new(addMrCommitUpdatedAt),
}
}
diff --git a/backend/plugins/gitlab/models/mr.go
b/backend/plugins/gitlab/models/mr.go
index 4d8e63ec5..9d8acae8d 100644
--- a/backend/plugins/gitlab/models/mr.go
+++ b/backend/plugins/gitlab/models/mr.go
@@ -53,6 +53,11 @@ type GitlabMergeRequest struct {
Component string `gorm:"type:varchar(255)"`
Additions *int `gorm:"comment:Lines added in this MR diff
only"`
Deletions *int `gorm:"comment:Lines deleted in this MR diff
only"`
+ // CommitUpdatedAt holds the most recent authored date among the
commits of this MR.
+ // GitLab does not bump `updated_at` for every push (e.g.
force-pushes), so this
+ // timestamp is used in addition to GitlabUpdatedAt to detect MRs that
changed
+ // during incremental collection.
+ CommitUpdatedAt *time.Time `gorm:"comment:Latest authored date of the
commits belonging to this MR"`
common.NoPKModel
}
diff --git a/backend/plugins/gitlab/tasks/mr_commit_extractor.go
b/backend/plugins/gitlab/tasks/mr_commit_extractor.go
index 7a94f9fee..506f0321b 100644
--- a/backend/plugins/gitlab/tasks/mr_commit_extractor.go
+++ b/backend/plugins/gitlab/tasks/mr_commit_extractor.go
@@ -104,7 +104,37 @@ func ExtractApiMergeRequestsCommits(subtaskCtx
plugin.SubTaskContext) errors.Err
return err
}
- return extractor.Execute()
+ if err := extractor.Execute(); err != nil {
+ return err
+ }
+
+ return updateMrCommitUpdatedAt(subtaskCtx)
+}
+
+// updateMrCommitUpdatedAt refreshes
`_tool_gitlab_merge_requests.commit_updated_at` with the
+// latest authored date of the MR's commits. GitLab does not always bump the
MR's own
+// `updated_at` when commits are pushed (e.g. force-pushes), so subtasks
collecting
+// MR sub-entities incrementally would otherwise skip those MRs.
+func updateMrCommitUpdatedAt(subtaskCtx plugin.SubTaskContext) errors.Error {
+ data := subtaskCtx.GetData().(*GitlabTaskData)
+ return subtaskCtx.GetDal().Exec(`
+ UPDATE _tool_gitlab_merge_requests AS gmr
+ SET commit_updated_at = (
+ SELECT MAX(gmc.commit_authored_date)
+ FROM _tool_gitlab_mr_commits gmc
+ WHERE gmc.connection_id = gmr.connection_id
+ AND gmc.merge_request_id = gmr.gitlab_id
+ )
+ WHERE gmr.connection_id = ?
+ AND gmr.project_id = ?
+ AND EXISTS (
+ SELECT 1
+ FROM _tool_gitlab_mr_commits gmc
+ WHERE gmc.connection_id = gmr.connection_id
+ AND gmc.merge_request_id = gmr.gitlab_id
+ )`,
+ data.Options.ConnectionId, data.Options.ProjectId,
+ )
}
// Convert the API response to our DB model instance
diff --git a/backend/plugins/gitlab/tasks/shared.go
b/backend/plugins/gitlab/tasks/shared.go
index eac88b66f..3462e2cb0 100644
--- a/backend/plugins/gitlab/tasks/shared.go
+++ b/backend/plugins/gitlab/tasks/shared.go
@@ -198,7 +198,8 @@ func GetMergeRequestsIterator(taskCtx
plugin.SubTaskContext, apiCollector *api.S
// Filter by the LATER of gitlab_updated_at or
commit_updated_at.
// Using only gitlab_updated_at misses MRs where new
commits were pushed
// without the MR itself being updated (e.g.
force-pushed commits).
- // COALESCE handles MRs with no recorded
commit_updated_at.
+ // commit_updated_at is maintained by the MR commit
extractor and is NULL
+ // for MRs without collected commits, hence the
COALESCE.
clauses = append(clauses, dal.Where(
`GREATEST(gmr.gitlab_updated_at,
COALESCE(gmr.commit_updated_at, gmr.gitlab_updated_at)) > ?`,
*apiCollector.GetSince(),
