This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 63b6a9b7b fix(gitlab): use stateful extractors for users and tags
(#9070)
63b6a9b7b is described below
commit 63b6a9b7b317e85867cb9bbccb970a869469b9bb
Author: Francisc Munteanu <[email protected]>
AuthorDate: Tue Sep 1 05:34:22 2026 +0200
fix(gitlab): use stateful extractors for users and tags (#9070)
NewApiExtractor always deleted tool-layer rows by raw params before insert,
so a truncated Collect Users run wiped accounts. NewStatefulApiExtractor
upserts incrementally, matching the rest of the GitLab plugin.
Co-authored-by: Cursor Grok 4.6 <[email protected]>
Co-authored-by: Cursor <[email protected]>
---
backend/plugins/gitlab/tasks/account_extractor.go | 61 +++++------------------
backend/plugins/gitlab/tasks/tag_extractor.go | 22 ++------
2 files changed, 18 insertions(+), 65 deletions(-)
diff --git a/backend/plugins/gitlab/tasks/account_extractor.go
b/backend/plugins/gitlab/tasks/account_extractor.go
index d2d24b48c..0024c3ead 100644
--- a/backend/plugins/gitlab/tasks/account_extractor.go
+++ b/backend/plugins/gitlab/tasks/account_extractor.go
@@ -18,7 +18,6 @@ limitations under the License.
package tasks
import (
- "encoding/json"
"strings"
"github.com/apache/incubator-devlake/core/errors"
@@ -41,52 +40,23 @@ var ExtractAccountsMeta = plugin.SubTaskMeta{
}
func ExtractAccounts(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_USER_TABLE)
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(taskCtx,
RAW_USER_TABLE)
// Do not extract createdUserAt if we are not using /users API
- var skipCreatedUserAt = strings.HasPrefix(data.ApiClient.GetEndpoint(),
"https://gitlab.com";)
-
- extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
- var userRes models.GitlabAccount
- err := errors.Convert(json.Unmarshal(row.Data,
&userRes))
- if err != nil {
- return nil, err
- }
+ skipCreatedUserAt := strings.HasPrefix(data.ApiClient.GetEndpoint(),
"https://gitlab.com";)
+ subtaskCommonArgs.SubtaskConfig = map[string]any{
+ "skipCreatedUserAt": skipCreatedUserAt,
+ }
- results := make([]interface{}, 0)
- var GitlabAccount *models.GitlabAccount
+ extractor, err :=
api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[models.GitlabAccount]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ Extract: func(userRes *models.GitlabAccount, _ *api.RawData)
([]interface{}, errors.Error) {
+ account := *userRes
+ account.ConnectionId = data.Options.ConnectionId
if skipCreatedUserAt {
- GitlabAccount = &models.GitlabAccount{
- ConnectionId:
data.Options.ConnectionId,
- GitlabId: userRes.GitlabId,
- Username: userRes.Username,
- Name: userRes.Name,
- State: userRes.State,
- MembershipState:
userRes.MembershipState,
- AvatarUrl: userRes.AvatarUrl,
- WebUrl: userRes.WebUrl,
- Email: userRes.Email,
- }
- } else {
- GitlabAccount = &models.GitlabAccount{
- ConnectionId:
data.Options.ConnectionId,
- GitlabId: userRes.GitlabId,
- Username: userRes.Username,
- Name: userRes.Name,
- State: userRes.State,
- MembershipState:
userRes.MembershipState,
- AvatarUrl: userRes.AvatarUrl,
- WebUrl: userRes.WebUrl,
- Email: userRes.Email,
- CreatedUserAt: userRes.CreatedUserAt,
- }
+ account.CreatedUserAt = nil
}
-
- results = append(results, GitlabAccount)
-
- return results, nil
+ return []interface{}{&account}, nil
},
})
@@ -94,10 +64,5 @@ func ExtractAccounts(taskCtx plugin.SubTaskContext)
errors.Error {
return err
}
- err = extractor.Execute()
- if err != nil {
- return err
- }
-
- return nil
+ return extractor.Execute()
}
diff --git a/backend/plugins/gitlab/tasks/tag_extractor.go
b/backend/plugins/gitlab/tasks/tag_extractor.go
index 60dac78a4..4f0a511de 100644
--- a/backend/plugins/gitlab/tasks/tag_extractor.go
+++ b/backend/plugins/gitlab/tasks/tag_extractor.go
@@ -18,8 +18,6 @@ limitations under the License.
package tasks
import (
- "encoding/json"
-
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -51,27 +49,17 @@ var ExtractTagMeta = plugin.SubTaskMeta{
}
func ExtractApiTag(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_TAG_TABLE)
-
- extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
- RawDataSubTaskArgs: *rawDataSubTaskArgs,
- Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
- // need to extract 1 kind of entities here
- results := make([]interface{}, 0, 1)
+ subtaskCommonArgs, data := CreateSubtaskCommonArgs(taskCtx,
RAW_TAG_TABLE)
- gitlabApiTag := &GitlabApiTag{}
- err := errors.Convert(json.Unmarshal(row.Data,
gitlabApiTag))
- if err != nil {
- return nil, err
- }
+ extractor, err :=
api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[GitlabApiTag]{
+ SubtaskCommonArgs: subtaskCommonArgs,
+ Extract: func(gitlabApiTag *GitlabApiTag, _ *api.RawData)
([]interface{}, errors.Error) {
gitlabTag, err := convertTag(gitlabApiTag)
if err != nil {
return nil, err
}
gitlabTag.ConnectionId = data.Options.ConnectionId
- results = append(results, gitlabTag)
-
- return results, nil
+ return []interface{}{gitlabTag}, nil
},
})
