This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new b63489c50 refactor: add subtask register (#4896)
b63489c50 is described below
commit b63489c502c6a7043680847b98a031c89aaca327
Author: chenggui53 <[email protected]>
AuthorDate: Thu Jun 8 16:01:57 2023 +0800
refactor: add subtask register (#4896)
* refactor: add subtask register
* refactor: add subtask register
* refactor: add subtaask register
---------
Co-authored-by: chenggui <[email protected]>
---
backend/core/plugin/plugin_task.go | 1 +
.../pluginhelper/subtaskmeta_sorter/dependency.go | 118 +++++++++++++++++++++
.../subtaskmeta_sorter/dependency_test.go | 98 +++++++++++++++++
.../pluginhelper/subtaskmeta_sorter/interface.go | 24 +++++
backend/plugins/gitlab/impl/impl.go | 50 +++------
backend/plugins/gitlab/tasks/account_collector.go | 5 +
backend/plugins/gitlab/tasks/account_convertor.go | 5 +
backend/plugins/gitlab/tasks/account_extractor.go | 5 +
backend/plugins/gitlab/tasks/commit_collector.go | 5 +
backend/plugins/gitlab/tasks/commit_convertor.go | 5 +
backend/plugins/gitlab/tasks/commit_extractor.go | 5 +
backend/plugins/gitlab/tasks/issue_collector.go | 5 +
backend/plugins/gitlab/tasks/issue_convertor.go | 5 +
backend/plugins/gitlab/tasks/issue_extractor.go | 5 +
.../plugins/gitlab/tasks/issue_label_convertor.go | 9 +-
backend/plugins/gitlab/tasks/job_collector.go | 5 +
backend/plugins/gitlab/tasks/job_convertor.go | 5 +
backend/plugins/gitlab/tasks/job_extractor.go | 5 +
backend/plugins/gitlab/tasks/mr_collector.go | 5 +
.../plugins/gitlab/tasks/mr_comment_convertor.go | 11 +-
.../plugins/gitlab/tasks/mr_commit_collector.go | 5 +
.../plugins/gitlab/tasks/mr_commit_convertor.go | 11 +-
.../plugins/gitlab/tasks/mr_commit_extractor.go | 5 +
backend/plugins/gitlab/tasks/mr_convertor.go | 5 +
.../plugins/gitlab/tasks/mr_detail_collector.go | 5 +
.../plugins/gitlab/tasks/mr_detail_extractor.go | 5 +
backend/plugins/gitlab/tasks/mr_enricher.go | 5 +
backend/plugins/gitlab/tasks/mr_extractor.go | 5 +
backend/plugins/gitlab/tasks/mr_label_convertor.go | 9 +-
backend/plugins/gitlab/tasks/mr_note_collector.go | 5 +
backend/plugins/gitlab/tasks/mr_note_extractor.go | 5 +
backend/plugins/gitlab/tasks/pipeline_collector.go | 5 +
.../gitlab/tasks/pipeline_commit_convertor.go | 5 +
backend/plugins/gitlab/tasks/pipeline_convertor.go | 5 +
.../gitlab/tasks/pipeline_detail_collector.go | 5 +
.../gitlab/tasks/pipeline_detail_extractor.go | 5 +
backend/plugins/gitlab/tasks/pipeline_extractor.go | 5 +
backend/plugins/gitlab/tasks/project_convertor.go | 5 +
backend/plugins/gitlab/tasks/register.go | 26 +++++
backend/plugins/gitlab/tasks/tag_collector.go | 5 +
backend/plugins/gitlab/tasks/tag_extractor.go | 5 +
41 files changed, 464 insertions(+), 48 deletions(-)
diff --git a/backend/core/plugin/plugin_task.go
b/backend/core/plugin/plugin_task.go
index 9bc595e49..9baf274e3 100644
--- a/backend/core/plugin/plugin_task.go
+++ b/backend/core/plugin/plugin_task.go
@@ -97,6 +97,7 @@ type SubTaskMeta struct {
EnabledByDefault bool
Description string
DomainTypes []string
+ Dependencies []*SubTaskMeta
}
// PluginTask Implement this interface to let framework run tasks for you
diff --git a/backend/helpers/pluginhelper/subtaskmeta_sorter/dependency.go
b/backend/helpers/pluginhelper/subtaskmeta_sorter/dependency.go
new file mode 100644
index 000000000..d310c1fe5
--- /dev/null
+++ b/backend/helpers/pluginhelper/subtaskmeta_sorter/dependency.go
@@ -0,0 +1,118 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package subtaskmeta_sorter
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "sort"
+)
+
+type DependencySorter struct {
+ metas []*plugin.SubTaskMeta
+}
+
+func NewDependencySorter(metas []*plugin.SubTaskMeta) SubTaskMetaSorter {
+ return &DependencySorter{metas: metas}
+}
+
+func (d *DependencySorter) Sort() ([]plugin.SubTaskMeta, error) {
+ return topologicalSort(d.metas)
+}
+
+// stable topological sort
+func topologicalSort(metas []*plugin.SubTaskMeta) ([]plugin.SubTaskMeta,
error) {
+ // which state will make a cycle
+ dependenciesMap := make(map[string][]string)
+ nameMetaMap := make(map[string]*plugin.SubTaskMeta)
+
+ for _, item := range metas {
+ if item != nil {
+ nameMetaMap[item.Name] = item
+ }
+
+ if _, ok := dependenciesMap[item.Name]; !ok {
+ if len(item.Dependencies) != 0 {
+ dependenciesMap[item.Name] = make([]string, 0)
+ for _, dependencyItem := range
item.Dependencies {
+ dependenciesMap[item.Name] =
append(dependenciesMap[item.Name], dependencyItem.Name)
+ }
+ } else {
+ dependenciesMap[item.Name] = make([]string, 0)
+ }
+ } else {
+ return nil, fmt.Errorf("duplicate subtaskmetas detected
in list: %s", item.Name)
+ }
+ }
+
+ orderedSubtaskList := make([]plugin.SubTaskMeta, 0)
+ for {
+ if len(dependenciesMap) == 0 {
+ break
+ }
+
+ tmpList := make([]string, 0)
+ for key, item := range dependenciesMap {
+ if len(item) == 0 {
+ tmpList = append(tmpList, key)
+ }
+ }
+ if len(tmpList) == 0 {
+ return nil, fmt.Errorf("cyclic dependency detected:
%v", dependenciesMap)
+ }
+
+ // remove item in dependencies map
+ for key, value := range dependenciesMap {
+ if contains(tmpList, key) {
+ delete(dependenciesMap, key)
+ } else {
+ dependenciesMap[key] = removeElements(value,
tmpList)
+ }
+ }
+
+ sort.Strings(tmpList)
+ // convert item to subtaskmeta by name, and append to
orderedSubtaskList
+ for _, item := range tmpList {
+ value, ok := nameMetaMap[item]
+ if !ok {
+ return nil, fmt.Errorf("illeagal subtaskmeta
detected %s", item)
+ }
+ orderedSubtaskList = append(orderedSubtaskList, *value)
+ }
+ }
+ return orderedSubtaskList, nil
+}
+
+func contains[T comparable](itemList []T, item T) bool {
+ for _, newItem := range itemList {
+ if item == newItem {
+ return true
+ }
+ }
+ return false
+}
+
+func removeElements[T comparable](raw, toRemove []T) []T {
+ newList := make([]T, 0)
+ for _, item := range raw {
+ if !contains(toRemove, item) {
+ newList = append(newList, item)
+ }
+ }
+ return newList
+}
diff --git a/backend/helpers/pluginhelper/subtaskmeta_sorter/dependency_test.go
b/backend/helpers/pluginhelper/subtaskmeta_sorter/dependency_test.go
new file mode 100644
index 000000000..924b3be79
--- /dev/null
+++ b/backend/helpers/pluginhelper/subtaskmeta_sorter/dependency_test.go
@@ -0,0 +1,98 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package subtaskmeta_sorter
+
+import (
+ "github.com/apache/incubator-devlake/core/plugin"
+ "reflect"
+ "testing"
+)
+
+func Test_topologicalSort(t *testing.T) {
+ pluginA := plugin.SubTaskMeta{
+ Name: "A",
+ Dependencies: []*plugin.SubTaskMeta{},
+ }
+ pluginB := plugin.SubTaskMeta{
+ Name: "B",
+ Dependencies: []*plugin.SubTaskMeta{&pluginA},
+ }
+ pluginC := plugin.SubTaskMeta{
+ Name: "C",
+ Dependencies: []*plugin.SubTaskMeta{&pluginA},
+ }
+ type args struct {
+ metas []*plugin.SubTaskMeta
+ }
+ tests := []struct {
+ name string
+ args args
+ want []plugin.SubTaskMeta
+ wantErr bool
+ }{
+ {
+ name: "correct stable order",
+ args: args{
+ metas: []*plugin.SubTaskMeta{&pluginA,
&pluginC, &pluginB},
+ },
+ want: []plugin.SubTaskMeta{
+ pluginA, pluginB, pluginC,
+ },
+ wantErr: false,
+ },
+ {
+ name: "duplicate error",
+ args: args{
+ metas: []*plugin.SubTaskMeta{&pluginA,
&pluginA, &pluginB},
+ },
+ want: nil,
+ wantErr: true,
+ },
+ {
+ name: "cycle error",
+ args: args{[]*plugin.SubTaskMeta{
+ &plugin.SubTaskMeta{
+ Name: "D",
+ Dependencies:
[]*plugin.SubTaskMeta{&plugin.SubTaskMeta{
+ Name: "E",
+ }},
+ },
+ &plugin.SubTaskMeta{
+ Name: "E",
+ Dependencies:
[]*plugin.SubTaskMeta{&plugin.SubTaskMeta{
+ Name: "D",
+ }},
+ },
+ }},
+ want: nil,
+ wantErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := topologicalSort(tt.args.metas)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("topologicalSort() error = %v, wantErr
%v", err, tt.wantErr)
+ return
+ }
+ if !reflect.DeepEqual(got, tt.want) {
+ t.Errorf("topologicalSort() got = %v, want %v",
got, tt.want)
+ }
+ })
+ }
+}
diff --git a/backend/helpers/pluginhelper/subtaskmeta_sorter/interface.go
b/backend/helpers/pluginhelper/subtaskmeta_sorter/interface.go
new file mode 100644
index 000000000..fb4e30b5e
--- /dev/null
+++ b/backend/helpers/pluginhelper/subtaskmeta_sorter/interface.go
@@ -0,0 +1,24 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package subtaskmeta_sorter
+
+import "github.com/apache/incubator-devlake/core/plugin"
+
+type SubTaskMetaSorter interface {
+ Sort() ([]plugin.SubTaskMeta, error)
+}
diff --git a/backend/plugins/gitlab/impl/impl.go
b/backend/plugins/gitlab/impl/impl.go
index 67e2d0d17..fd7bb02e6 100644
--- a/backend/plugins/gitlab/impl/impl.go
+++ b/backend/plugins/gitlab/impl/impl.go
@@ -19,6 +19,7 @@ package impl
import (
"fmt"
+
"github.com/apache/incubator-devlake/helpers/pluginhelper/subtaskmeta_sorter"
"time"
"github.com/apache/incubator-devlake/core/context"
@@ -46,6 +47,13 @@ var _ interface {
type Gitlab string
+func init() {
+ // check subtask meta loop when init subtask meta
+ if _, err :=
subtaskmeta_sorter.NewDependencySorter(tasks.SubTaskMetaList).Sort(); err !=
nil {
+ panic(err)
+ }
+}
+
func (p Gitlab) Init(basicRes context.BasicRes) errors.Error {
api.Init(basicRes)
return nil
@@ -94,45 +102,11 @@ func (p Gitlab) Description() string {
}
func (p Gitlab) SubTaskMetas() []plugin.SubTaskMeta {
- return []plugin.SubTaskMeta{
- tasks.CollectApiIssuesMeta,
- tasks.ExtractApiIssuesMeta,
- tasks.CollectApiMergeRequestsMeta,
- tasks.ExtractApiMergeRequestsMeta,
- tasks.CollectApiMergeRequestDetailsMeta,
- tasks.CollectApiMergeRequestDetailsMeta,
- tasks.CollectApiMrNotesMeta,
- tasks.ExtractApiMrNotesMeta,
- tasks.CollectApiMrCommitsMeta,
- tasks.ExtractApiMrCommitsMeta,
- tasks.CollectApiPipelinesMeta,
- tasks.ExtractApiPipelinesMeta,
- tasks.CollectApiPipelineDetailsMeta,
- tasks.ExtractApiPipelineDetailsMeta,
- tasks.CollectApiJobsMeta,
- tasks.ExtractApiJobsMeta,
- tasks.EnrichMergeRequestsMeta,
- tasks.CollectAccountsMeta,
- tasks.ExtractAccountsMeta,
- tasks.ConvertAccountsMeta,
- tasks.ConvertProjectMeta,
- tasks.ConvertApiMergeRequestsMeta,
- tasks.ConvertMrCommentMeta,
- tasks.ConvertApiMrCommitsMeta,
- tasks.ConvertIssuesMeta,
- tasks.ConvertIssueAssigneeMeta,
- tasks.ConvertIssueLabelsMeta,
- tasks.ConvertMrLabelsMeta,
- tasks.ConvertCommitsMeta,
- tasks.ConvertPipelineMeta,
- tasks.ConvertPipelineCommitMeta,
- tasks.ConvertJobMeta,
- tasks.CollectApiCommitsMeta,
- tasks.ExtractApiCommitsMeta,
- tasks.ExtractApiMergeRequestDetailsMeta,
- tasks.CollectTagMeta,
- tasks.ExtractTagMeta,
+ list, err :=
subtaskmeta_sorter.NewDependencySorter(tasks.SubTaskMetaList).Sort()
+ if err != nil {
+ panic(err)
}
+ return list
}
func (p Gitlab) PrepareTaskData(taskCtx plugin.TaskContext, options
map[string]interface{}) (interface{}, errors.Error) {
diff --git a/backend/plugins/gitlab/tasks/account_collector.go
b/backend/plugins/gitlab/tasks/account_collector.go
index 43142938d..fb21f2b9e 100644
--- a/backend/plugins/gitlab/tasks/account_collector.go
+++ b/backend/plugins/gitlab/tasks/account_collector.go
@@ -30,6 +30,10 @@ import (
"golang.org/x/mod/semver"
)
+func init() {
+ RegisterSubtaskMeta(&CollectAccountsMeta)
+}
+
const RAW_USER_TABLE = "gitlab_api_users"
var CollectAccountsMeta = plugin.SubTaskMeta{
@@ -38,6 +42,7 @@ var CollectAccountsMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "collect gitlab users, does not support either
timeFilter or diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
+ Dependencies: []*plugin.SubTaskMeta{&EnrichMergeRequestsMeta},
}
func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/gitlab/tasks/account_convertor.go
b/backend/plugins/gitlab/tasks/account_convertor.go
index 2a5799979..36bc314c8 100644
--- a/backend/plugins/gitlab/tasks/account_convertor.go
+++ b/backend/plugins/gitlab/tasks/account_convertor.go
@@ -29,12 +29,17 @@ import (
"reflect"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertAccountsMeta)
+}
+
var ConvertAccountsMeta = plugin.SubTaskMeta{
Name: "convertAccounts",
EntryPoint: ConvertAccounts,
EnabledByDefault: true,
Description: "Convert tool layer table gitlab_users into domain
layer table accounts",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
+ Dependencies: []*plugin.SubTaskMeta{&ExtractAccountsMeta},
}
func ConvertAccounts(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/gitlab/tasks/account_extractor.go
b/backend/plugins/gitlab/tasks/account_extractor.go
index 56df7fdbb..1b57a7b2a 100644
--- a/backend/plugins/gitlab/tasks/account_extractor.go
+++ b/backend/plugins/gitlab/tasks/account_extractor.go
@@ -26,12 +26,17 @@ import (
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractAccountsMeta)
+}
+
var ExtractAccountsMeta = plugin.SubTaskMeta{
Name: "extractAccounts",
EntryPoint: ExtractAccounts,
EnabledByDefault: true,
Description: "Extract raw workspace data into tool layer table
_tool_gitlab_accounts",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
+ Dependencies: []*plugin.SubTaskMeta{&CollectAccountsMeta},
}
func ExtractAccounts(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/gitlab/tasks/commit_collector.go
b/backend/plugins/gitlab/tasks/commit_collector.go
index 90663bb5a..b97744129 100644
--- a/backend/plugins/gitlab/tasks/commit_collector.go
+++ b/backend/plugins/gitlab/tasks/commit_collector.go
@@ -23,6 +23,10 @@ import (
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
+func init() {
+ RegisterSubtaskMeta(&CollectApiCommitsMeta)
+}
+
const RAW_COMMIT_TABLE = "gitlab_api_commit"
var CollectApiCommitsMeta = plugin.SubTaskMeta{
@@ -31,6 +35,7 @@ var CollectApiCommitsMeta = plugin.SubTaskMeta{
EnabledByDefault: false,
Description: "Collect commit data from gitlab api, does not
support either timeFilter or diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
+ Dependencies: []*plugin.SubTaskMeta{&ConvertJobMeta},
}
type GitlabApiCommit struct {
diff --git a/backend/plugins/gitlab/tasks/commit_convertor.go
b/backend/plugins/gitlab/tasks/commit_convertor.go
index 0450a016e..c285a785e 100644
--- a/backend/plugins/gitlab/tasks/commit_convertor.go
+++ b/backend/plugins/gitlab/tasks/commit_convertor.go
@@ -28,12 +28,17 @@ import (
"reflect"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertCommitsMeta)
+}
+
var ConvertCommitsMeta = plugin.SubTaskMeta{
Name: "convertApiCommits",
EntryPoint: ConvertApiCommits,
EnabledByDefault: false,
Description: "Update domain layer commit according to
GitlabCommit",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
+ Dependencies: []*plugin.SubTaskMeta{&ConvertMrLabelsMeta},
}
func ConvertApiCommits(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/gitlab/tasks/commit_extractor.go
b/backend/plugins/gitlab/tasks/commit_extractor.go
index d2765e65d..7cbacf997 100644
--- a/backend/plugins/gitlab/tasks/commit_extractor.go
+++ b/backend/plugins/gitlab/tasks/commit_extractor.go
@@ -25,12 +25,17 @@ import (
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractApiCommitsMeta)
+}
+
var ExtractApiCommitsMeta = plugin.SubTaskMeta{
Name: "extractApiCommits",
EntryPoint: ExtractApiCommits,
EnabledByDefault: false,
Description: "Extract raw commit data into tool layer table
GitlabCommit,GitlabAccount and GitlabProjectCommit",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
+ Dependencies: []*plugin.SubTaskMeta{&CollectApiCommitsMeta},
}
func ExtractApiCommits(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/gitlab/tasks/issue_collector.go
b/backend/plugins/gitlab/tasks/issue_collector.go
index faecc6cc3..a8b51c5c6 100644
--- a/backend/plugins/gitlab/tasks/issue_collector.go
+++ b/backend/plugins/gitlab/tasks/issue_collector.go
@@ -29,6 +29,10 @@ import (
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
+func init() {
+ RegisterSubtaskMeta(&CollectApiIssuesMeta)
+}
+
const RAW_ISSUE_TABLE = "gitlab_api_issues"
var CollectApiIssuesMeta = plugin.SubTaskMeta{
@@ -37,6 +41,7 @@ var CollectApiIssuesMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Collect issues data from Gitlab api, supports both
timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+ Dependencies: []*plugin.SubTaskMeta{},
}
func CollectApiIssues(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/gitlab/tasks/issue_convertor.go
b/backend/plugins/gitlab/tasks/issue_convertor.go
index 3da4c5173..6d09b4b19 100644
--- a/backend/plugins/gitlab/tasks/issue_convertor.go
+++ b/backend/plugins/gitlab/tasks/issue_convertor.go
@@ -31,12 +31,17 @@ import (
"strings"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertIssuesMeta)
+}
+
var ConvertIssuesMeta = plugin.SubTaskMeta{
Name: "convertIssues",
EntryPoint: ConvertIssues,
EnabledByDefault: true,
Description: "Convert tool layer table gitlab_issues into domain
layer table issues",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+ Dependencies: []*plugin.SubTaskMeta{&ConvertApiMrCommitsMeta},
}
func ConvertIssues(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/gitlab/tasks/issue_extractor.go
b/backend/plugins/gitlab/tasks/issue_extractor.go
index 793c8edf2..5b36ae231 100644
--- a/backend/plugins/gitlab/tasks/issue_extractor.go
+++ b/backend/plugins/gitlab/tasks/issue_extractor.go
@@ -28,12 +28,17 @@ import (
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractApiIssuesMeta)
+}
+
var ExtractApiIssuesMeta = plugin.SubTaskMeta{
Name: "extractApiIssues",
EntryPoint: ExtractApiIssues,
EnabledByDefault: true,
Description: "Extract raw Issues data into tool layer table
gitlab_issues",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+ Dependencies: []*plugin.SubTaskMeta{&CollectApiIssuesMeta},
}
type IssuesResponse struct {
diff --git a/backend/plugins/gitlab/tasks/issue_label_convertor.go
b/backend/plugins/gitlab/tasks/issue_label_convertor.go
index c22c2688d..c79ce1590 100644
--- a/backend/plugins/gitlab/tasks/issue_label_convertor.go
+++ b/backend/plugins/gitlab/tasks/issue_label_convertor.go
@@ -28,12 +28,17 @@ import (
"reflect"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertIssueLabelsMeta)
+}
+
var ConvertIssueLabelsMeta = plugin.SubTaskMeta{
Name: "convertIssueLabels",
EntryPoint: ConvertIssueLabels,
EnabledByDefault: true,
Description: "Convert tool layer table gitlab_issue_labels into
domain layer table issue_labels",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+ Dependencies: []*plugin.SubTaskMeta{&ConvertIssuesMeta},
}
func ConvertIssueLabels(taskCtx plugin.SubTaskContext) errors.Error {
@@ -43,9 +48,9 @@ func ConvertIssueLabels(taskCtx plugin.SubTaskContext)
errors.Error {
clauses := []dal.Clause{
dal.Select("*"),
dal.From(&models.GitlabIssueLabel{}),
- dal.Join(`left join _tool_gitlab_issues on
+ dal.Join(`left join _tool_gitlab_issues on
_tool_gitlab_issues.gitlab_id =
_tool_gitlab_issue_labels.issue_id`),
- dal.Where(`_tool_gitlab_issues.project_id = ?
+ dal.Where(`_tool_gitlab_issues.project_id = ?
and _tool_gitlab_issues.connection_id = ?`,
projectId, data.Options.ConnectionId),
dal.Orderby("issue_id ASC"),
diff --git a/backend/plugins/gitlab/tasks/job_collector.go
b/backend/plugins/gitlab/tasks/job_collector.go
index 8fc5d369b..948468a49 100644
--- a/backend/plugins/gitlab/tasks/job_collector.go
+++ b/backend/plugins/gitlab/tasks/job_collector.go
@@ -33,6 +33,10 @@ import (
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
+func init() {
+ RegisterSubtaskMeta(&CollectApiJobsMeta)
+}
+
const RAW_JOB_TABLE = "gitlab_api_job"
type SimpleGitlabApiJob struct {
@@ -46,6 +50,7 @@ var CollectApiJobsMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Collect job data from gitlab api, supports both
timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+ Dependencies: []*plugin.SubTaskMeta{&ExtractApiPipelineDetailsMeta},
}
func CollectApiJobs(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/gitlab/tasks/job_convertor.go
b/backend/plugins/gitlab/tasks/job_convertor.go
index 32d876017..d0e0a750a 100644
--- a/backend/plugins/gitlab/tasks/job_convertor.go
+++ b/backend/plugins/gitlab/tasks/job_convertor.go
@@ -30,12 +30,17 @@ import (
gitlabModels "github.com/apache/incubator-devlake/plugins/gitlab/models"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertJobMeta)
+}
+
var ConvertJobMeta = plugin.SubTaskMeta{
Name: "convertJobs",
EntryPoint: ConvertJobs,
EnabledByDefault: true,
Description: "Convert tool layer table gitlab_job into domain
layer table job",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+ Dependencies: []*plugin.SubTaskMeta{&ConvertPipelineCommitMeta},
}
func ConvertJobs(taskCtx plugin.SubTaskContext) (err errors.Error) {
diff --git a/backend/plugins/gitlab/tasks/job_extractor.go
b/backend/plugins/gitlab/tasks/job_extractor.go
index f06158911..bbf795a51 100644
--- a/backend/plugins/gitlab/tasks/job_extractor.go
+++ b/backend/plugins/gitlab/tasks/job_extractor.go
@@ -25,6 +25,10 @@ import (
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractApiJobsMeta)
+}
+
type ApiJob struct {
Id int `json:"id"`
Status string
@@ -50,6 +54,7 @@ var ExtractApiJobsMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Extract raw GitlabJob data into tool layer table
GitlabPipeline",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+ Dependencies: []*plugin.SubTaskMeta{&CollectApiJobsMeta},
}
func ExtractApiJobs(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/gitlab/tasks/mr_collector.go
b/backend/plugins/gitlab/tasks/mr_collector.go
index a9a0920b1..125262e60 100644
--- a/backend/plugins/gitlab/tasks/mr_collector.go
+++ b/backend/plugins/gitlab/tasks/mr_collector.go
@@ -28,12 +28,17 @@ import (
const RAW_MERGE_REQUEST_TABLE = "gitlab_api_merge_requests"
+func init() {
+ RegisterSubtaskMeta(&CollectApiMergeRequestsMeta)
+}
+
var CollectApiMergeRequestsMeta = plugin.SubTaskMeta{
Name: "collectApiMergeRequests",
EntryPoint: CollectApiMergeRequests,
EnabledByDefault: true,
Description: "Collect merge requests data from gitlab api,
supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
+ Dependencies: []*plugin.SubTaskMeta{&ExtractApiIssuesMeta},
}
func CollectApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/gitlab/tasks/mr_comment_convertor.go
b/backend/plugins/gitlab/tasks/mr_comment_convertor.go
index dd376d5d2..1ea38c9c6 100644
--- a/backend/plugins/gitlab/tasks/mr_comment_convertor.go
+++ b/backend/plugins/gitlab/tasks/mr_comment_convertor.go
@@ -29,12 +29,17 @@ import (
"reflect"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertMrCommentMeta)
+}
+
var ConvertMrCommentMeta = plugin.SubTaskMeta{
Name: "convertMergeRequestComment",
EntryPoint: ConvertMergeRequestComment,
EnabledByDefault: true,
Description: "Add domain layer Comment according to
GitlabMrComment",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
+ Dependencies: []*plugin.SubTaskMeta{&ConvertApiMergeRequestsMeta},
}
func ConvertMergeRequestComment(taskCtx plugin.SubTaskContext) errors.Error {
@@ -42,10 +47,10 @@ func ConvertMergeRequestComment(taskCtx
plugin.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
clauses := []dal.Clause{
dal.From(&models.GitlabMrComment{}),
- dal.Join(`left join _tool_gitlab_merge_requests on
- _tool_gitlab_merge_requests.gitlab_id =
+ dal.Join(`left join _tool_gitlab_merge_requests on
+ _tool_gitlab_merge_requests.gitlab_id =
_tool_gitlab_mr_comments.merge_request_id`),
- dal.Where(`_tool_gitlab_merge_requests.project_id = ?
+ dal.Where(`_tool_gitlab_merge_requests.project_id = ?
and _tool_gitlab_mr_comments.connection_id = ?`,
data.Options.ProjectId, data.Options.ConnectionId),
}
diff --git a/backend/plugins/gitlab/tasks/mr_commit_collector.go
b/backend/plugins/gitlab/tasks/mr_commit_collector.go
index a6f50cf45..f0e4350ba 100644
--- a/backend/plugins/gitlab/tasks/mr_commit_collector.go
+++ b/backend/plugins/gitlab/tasks/mr_commit_collector.go
@@ -23,6 +23,10 @@ import (
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
+func init() {
+ RegisterSubtaskMeta(&CollectApiMrCommitsMeta)
+}
+
const RAW_MERGE_REQUEST_COMMITS_TABLE = "gitlab_api_merge_request_commits"
var CollectApiMrCommitsMeta = plugin.SubTaskMeta{
@@ -31,6 +35,7 @@ var CollectApiMrCommitsMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Collect merge requests commits data from gitlab api,
supports timeFilter but not diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
+ Dependencies: []*plugin.SubTaskMeta{&ExtractApiMrNotesMeta},
}
func CollectApiMergeRequestsCommits(taskCtx plugin.SubTaskContext)
errors.Error {
diff --git a/backend/plugins/gitlab/tasks/mr_commit_convertor.go
b/backend/plugins/gitlab/tasks/mr_commit_convertor.go
index 99b2a2b41..751e84a96 100644
--- a/backend/plugins/gitlab/tasks/mr_commit_convertor.go
+++ b/backend/plugins/gitlab/tasks/mr_commit_convertor.go
@@ -28,12 +28,17 @@ import (
"reflect"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertApiMrCommitsMeta)
+}
+
var ConvertApiMrCommitsMeta = plugin.SubTaskMeta{
Name: "convertApiMergeRequestsCommits",
EntryPoint: ConvertApiMergeRequestsCommits,
EnabledByDefault: true,
Description: "Add domain layer PullRequestCommit according to
GitlabMrCommit",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
+ Dependencies: []*plugin.SubTaskMeta{&ConvertMrCommentMeta},
}
func ConvertApiMergeRequestsCommits(taskCtx plugin.SubTaskContext)
errors.Error {
@@ -42,10 +47,10 @@ func ConvertApiMergeRequestsCommits(taskCtx
plugin.SubTaskContext) errors.Error
clauses := []dal.Clause{
dal.From(&models.GitlabMrCommit{}),
- dal.Join(`left join _tool_gitlab_merge_requests
- on _tool_gitlab_merge_requests.gitlab_id =
+ dal.Join(`left join _tool_gitlab_merge_requests
+ on _tool_gitlab_merge_requests.gitlab_id =
_tool_gitlab_mr_commits.merge_request_id`),
- dal.Where(`_tool_gitlab_merge_requests.project_id = ?
+ dal.Where(`_tool_gitlab_merge_requests.project_id = ?
and _tool_gitlab_merge_requests.connection_id = ?`,
data.Options.ProjectId, data.Options.ConnectionId),
dal.Orderby("merge_request_id ASC"),
diff --git a/backend/plugins/gitlab/tasks/mr_commit_extractor.go
b/backend/plugins/gitlab/tasks/mr_commit_extractor.go
index e237add9f..f71107532 100644
--- a/backend/plugins/gitlab/tasks/mr_commit_extractor.go
+++ b/backend/plugins/gitlab/tasks/mr_commit_extractor.go
@@ -25,12 +25,17 @@ import (
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractApiMrCommitsMeta)
+}
+
var ExtractApiMrCommitsMeta = plugin.SubTaskMeta{
Name: "extractApiMergeRequestsCommits",
EntryPoint: ExtractApiMergeRequestsCommits,
EnabledByDefault: true,
Description: "Extract raw merge requests commit data into tool
layer table GitlabMrCommit and GitlabCommit",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
+ Dependencies: []*plugin.SubTaskMeta{&CollectApiMrCommitsMeta},
}
func ExtractApiMergeRequestsCommits(taskCtx plugin.SubTaskContext)
errors.Error {
diff --git a/backend/plugins/gitlab/tasks/mr_convertor.go
b/backend/plugins/gitlab/tasks/mr_convertor.go
index b0fb831d7..89aa2c9ba 100644
--- a/backend/plugins/gitlab/tasks/mr_convertor.go
+++ b/backend/plugins/gitlab/tasks/mr_convertor.go
@@ -30,12 +30,17 @@ import (
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertApiMergeRequestsMeta)
+}
+
var ConvertApiMergeRequestsMeta = plugin.SubTaskMeta{
Name: "convertApiMergeRequests",
EntryPoint: ConvertApiMergeRequests,
EnabledByDefault: true,
Description: "Add domain layer PullRequest according to
GitlabMergeRequest",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
+ Dependencies: []*plugin.SubTaskMeta{&ConvertProjectMeta},
}
func ConvertApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/gitlab/tasks/mr_detail_collector.go
b/backend/plugins/gitlab/tasks/mr_detail_collector.go
index 5e1e6b74e..451a9391d 100644
--- a/backend/plugins/gitlab/tasks/mr_detail_collector.go
+++ b/backend/plugins/gitlab/tasks/mr_detail_collector.go
@@ -27,6 +27,10 @@ import (
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
+func init() {
+ RegisterSubtaskMeta(&CollectApiMergeRequestDetailsMeta)
+}
+
const RAW_MERGE_REQUEST_DETAIL_TABLE = "gitlab_api_merge_request_details"
var CollectApiMergeRequestDetailsMeta = plugin.SubTaskMeta{
@@ -35,6 +39,7 @@ var CollectApiMergeRequestDetailsMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Collect merge request Details data from gitlab api,
supports timeFilter but not diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
+ Dependencies: []*plugin.SubTaskMeta{&ExtractApiMergeRequestsMeta},
}
func CollectApiMergeRequestDetails(taskCtx plugin.SubTaskContext) errors.Error
{
diff --git a/backend/plugins/gitlab/tasks/mr_detail_extractor.go
b/backend/plugins/gitlab/tasks/mr_detail_extractor.go
index e48d88fb1..8a590d53e 100644
--- a/backend/plugins/gitlab/tasks/mr_detail_extractor.go
+++ b/backend/plugins/gitlab/tasks/mr_detail_extractor.go
@@ -27,12 +27,17 @@ import (
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractApiMergeRequestDetailsMeta)
+}
+
var ExtractApiMergeRequestDetailsMeta = plugin.SubTaskMeta{
Name: "extractApiMergeRequestDetails",
EntryPoint: ExtractApiMergeRequestDetails,
EnabledByDefault: true,
Description: "Extract raw merge request Details data into tool
layer table GitlabMergeRequest and GitlabReviewer",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
+ Dependencies: []*plugin.SubTaskMeta{&ExtractApiCommitsMeta},
}
func ExtractApiMergeRequestDetails(taskCtx plugin.SubTaskContext) errors.Error
{
diff --git a/backend/plugins/gitlab/tasks/mr_enricher.go
b/backend/plugins/gitlab/tasks/mr_enricher.go
index df3036533..256dffb7e 100644
--- a/backend/plugins/gitlab/tasks/mr_enricher.go
+++ b/backend/plugins/gitlab/tasks/mr_enricher.go
@@ -27,12 +27,17 @@ import (
"time"
)
+func init() {
+ RegisterSubtaskMeta(&EnrichMergeRequestsMeta)
+}
+
var EnrichMergeRequestsMeta = plugin.SubTaskMeta{
Name: "enrichMrs",
EntryPoint: EnrichMergeRequests,
EnabledByDefault: true,
Description: "Enrich merge requests data from GitlabCommit,
GitlabMrNote and GitlabMergeRequest",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
+ Dependencies: []*plugin.SubTaskMeta{&ExtractApiJobsMeta},
}
func EnrichMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/gitlab/tasks/mr_extractor.go
b/backend/plugins/gitlab/tasks/mr_extractor.go
index bb22570b5..8fe0ae8d2 100644
--- a/backend/plugins/gitlab/tasks/mr_extractor.go
+++ b/backend/plugins/gitlab/tasks/mr_extractor.go
@@ -28,6 +28,10 @@ import (
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractApiMergeRequestsMeta)
+}
+
type MergeRequestRes struct {
GitlabId int `json:"id"`
Iid int
@@ -75,6 +79,7 @@ var ExtractApiMergeRequestsMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Extract raw merge requests data into tool layer
table GitlabMergeRequest and GitlabReviewer",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
+ Dependencies: []*plugin.SubTaskMeta{&CollectApiMergeRequestsMeta},
}
func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/gitlab/tasks/mr_label_convertor.go
b/backend/plugins/gitlab/tasks/mr_label_convertor.go
index 84b951923..603332438 100644
--- a/backend/plugins/gitlab/tasks/mr_label_convertor.go
+++ b/backend/plugins/gitlab/tasks/mr_label_convertor.go
@@ -28,12 +28,17 @@ import (
"reflect"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertMrLabelsMeta)
+}
+
var ConvertMrLabelsMeta = plugin.SubTaskMeta{
Name: "convertMrLabels",
EntryPoint: ConvertMrLabels,
EnabledByDefault: true,
Description: "Convert tool layer table gitlab_mr_labels into
domain layer table pull_request_labels",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
+ Dependencies: []*plugin.SubTaskMeta{&ConvertIssueLabelsMeta},
}
func ConvertMrLabels(taskCtx plugin.SubTaskContext) errors.Error {
@@ -43,9 +48,9 @@ func ConvertMrLabels(taskCtx plugin.SubTaskContext)
errors.Error {
clauses := []dal.Clause{
dal.Select("*"),
dal.From(&models.GitlabMrLabel{}),
- dal.Join(`left join _tool_gitlab_merge_requests on
+ dal.Join(`left join _tool_gitlab_merge_requests on
_tool_gitlab_merge_requests.gitlab_id =
_tool_gitlab_mr_labels.mr_id`),
- dal.Where(`_tool_gitlab_merge_requests.project_id = ?
+ dal.Where(`_tool_gitlab_merge_requests.project_id = ?
and _tool_gitlab_merge_requests.connection_id = ?`,
projectId, data.Options.ConnectionId),
dal.Orderby("mr_id ASC"),
diff --git a/backend/plugins/gitlab/tasks/mr_note_collector.go
b/backend/plugins/gitlab/tasks/mr_note_collector.go
index 882dfc506..ece1d47d5 100644
--- a/backend/plugins/gitlab/tasks/mr_note_collector.go
+++ b/backend/plugins/gitlab/tasks/mr_note_collector.go
@@ -23,6 +23,10 @@ import (
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
+func init() {
+ RegisterSubtaskMeta(&CollectApiMrNotesMeta)
+}
+
const RAW_MERGE_REQUEST_NOTES_TABLE = "gitlab_api_merge_request_notes"
var CollectApiMrNotesMeta = plugin.SubTaskMeta{
@@ -31,6 +35,7 @@ var CollectApiMrNotesMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Collect merge requests notes data from gitlab api,
supports timeFilter but not diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
+ Dependencies:
[]*plugin.SubTaskMeta{&CollectApiMergeRequestDetailsMeta},
}
func CollectApiMergeRequestsNotes(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/gitlab/tasks/mr_note_extractor.go
b/backend/plugins/gitlab/tasks/mr_note_extractor.go
index ae7ab0814..80f01e708 100644
--- a/backend/plugins/gitlab/tasks/mr_note_extractor.go
+++ b/backend/plugins/gitlab/tasks/mr_note_extractor.go
@@ -25,6 +25,10 @@ import (
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractApiMrNotesMeta)
+}
+
type MergeRequestNote struct {
GitlabId int `json:"id"`
MergeRequestId int `json:"noteable_id"`
@@ -48,6 +52,7 @@ var ExtractApiMrNotesMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Extract raw merge requests notes data into tool
layer table GitlabMrNote",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
+ Dependencies: []*plugin.SubTaskMeta{&CollectApiMrNotesMeta},
}
func ExtractApiMergeRequestsNotes(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/gitlab/tasks/pipeline_collector.go
b/backend/plugins/gitlab/tasks/pipeline_collector.go
index 8eb589617..1ef244887 100644
--- a/backend/plugins/gitlab/tasks/pipeline_collector.go
+++ b/backend/plugins/gitlab/tasks/pipeline_collector.go
@@ -27,6 +27,10 @@ import (
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
+func init() {
+ RegisterSubtaskMeta(&CollectApiPipelinesMeta)
+}
+
const RAW_PIPELINE_TABLE = "gitlab_api_pipeline"
var CollectApiPipelinesMeta = plugin.SubTaskMeta{
@@ -35,6 +39,7 @@ var CollectApiPipelinesMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Collect pipeline data from gitlab api, supports both
timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+ Dependencies: []*plugin.SubTaskMeta{&ExtractApiMrCommitsMeta},
}
func CollectApiPipelines(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go
b/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go
index f400de05d..cd5d831a8 100644
--- a/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go
@@ -29,12 +29,17 @@ import (
gitlabModels "github.com/apache/incubator-devlake/plugins/gitlab/models"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertPipelineCommitMeta)
+}
+
var ConvertPipelineCommitMeta = plugin.SubTaskMeta{
Name: "convertPipelineCommits",
EntryPoint: ConvertPipelineCommits,
EnabledByDefault: true,
Description: "Convert tool layer table gitlab_pipeline_project
into domain layer table pipeline",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+ Dependencies: []*plugin.SubTaskMeta{&ConvertPipelineMeta},
}
func ConvertPipelineCommits(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/gitlab/tasks/pipeline_convertor.go
b/backend/plugins/gitlab/tasks/pipeline_convertor.go
index 30e6b979b..ea871a012 100644
--- a/backend/plugins/gitlab/tasks/pipeline_convertor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_convertor.go
@@ -32,12 +32,17 @@ import (
gitlabModels "github.com/apache/incubator-devlake/plugins/gitlab/models"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertPipelineMeta)
+}
+
var ConvertPipelineMeta = plugin.SubTaskMeta{
Name: "convertPipelines",
EntryPoint: ConvertPipelines,
EnabledByDefault: true,
Description: "Convert tool layer table gitlab_pipeline into domain
layer table pipeline",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+ Dependencies: []*plugin.SubTaskMeta{&ConvertCommitsMeta},
}
func ConvertPipelines(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_collector.go
b/backend/plugins/gitlab/tasks/pipeline_detail_collector.go
index 0d22c81ac..856659adb 100644
--- a/backend/plugins/gitlab/tasks/pipeline_detail_collector.go
+++ b/backend/plugins/gitlab/tasks/pipeline_detail_collector.go
@@ -28,6 +28,10 @@ import (
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
+func init() {
+ RegisterSubtaskMeta(&CollectApiPipelineDetailsMeta)
+}
+
const RAW_PIPELINE_DETAILS_TABLE = "gitlab_api_pipeline_details"
var CollectApiPipelineDetailsMeta = plugin.SubTaskMeta{
@@ -36,6 +40,7 @@ var CollectApiPipelineDetailsMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Collect pipeline details data from gitlab api,
supports both timeFilter and diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+ Dependencies: []*plugin.SubTaskMeta{&ExtractApiPipelinesMeta},
}
func CollectApiPipelineDetails(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
index 1579eac8b..7370ad213 100644
--- a/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go
@@ -27,12 +27,17 @@ import (
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractApiPipelineDetailsMeta)
+}
+
var ExtractApiPipelineDetailsMeta = plugin.SubTaskMeta{
Name: "extractApiPipelineDetails",
EntryPoint: ExtractApiPipelineDetails,
EnabledByDefault: true,
Description: "Extract raw pipeline details data into tool layer
table GitlabPipeline",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+ Dependencies: []*plugin.SubTaskMeta{&CollectApiPipelineDetailsMeta},
}
func ExtractApiPipelineDetails(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/gitlab/tasks/pipeline_extractor.go
b/backend/plugins/gitlab/tasks/pipeline_extractor.go
index 0c275a7d5..7b1baef35 100644
--- a/backend/plugins/gitlab/tasks/pipeline_extractor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_extractor.go
@@ -27,6 +27,10 @@ import (
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractApiPipelinesMeta)
+}
+
type ApiDetailedStatus struct {
Icon string
Text string
@@ -61,6 +65,7 @@ var ExtractApiPipelinesMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Extract raw pipelines data into tool layer table
GitlabPipeline",
DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+ Dependencies: []*plugin.SubTaskMeta{&CollectApiPipelinesMeta},
}
func ExtractApiPipelines(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/gitlab/tasks/project_convertor.go
b/backend/plugins/gitlab/tasks/project_convertor.go
index 419f1073b..f4990f99f 100644
--- a/backend/plugins/gitlab/tasks/project_convertor.go
+++ b/backend/plugins/gitlab/tasks/project_convertor.go
@@ -33,6 +33,10 @@ import (
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
+func init() {
+ RegisterSubtaskMeta(&ConvertProjectMeta)
+}
+
const RAW_PROJECT_TABLE = "gitlab_api_project"
type GitlabApiProject struct {
@@ -58,6 +62,7 @@ var ConvertProjectMeta = plugin.SubTaskMeta{
EnabledByDefault: true,
Description: "Add domain layer Repo according to GitlabProject",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE,
plugin.DOMAIN_TYPE_TICKET},
+ Dependencies: []*plugin.SubTaskMeta{&ConvertAccountsMeta},
}
func ConvertApiProjects(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/gitlab/tasks/register.go
b/backend/plugins/gitlab/tasks/register.go
new file mode 100644
index 000000000..51b12a76c
--- /dev/null
+++ b/backend/plugins/gitlab/tasks/register.go
@@ -0,0 +1,26 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import "github.com/apache/incubator-devlake/core/plugin"
+
+var SubTaskMetaList []*plugin.SubTaskMeta
+
+func RegisterSubtaskMeta(meta *plugin.SubTaskMeta) {
+ SubTaskMetaList = append(SubTaskMetaList, meta)
+}
diff --git a/backend/plugins/gitlab/tasks/tag_collector.go
b/backend/plugins/gitlab/tasks/tag_collector.go
index d36a429e4..af5e518ae 100644
--- a/backend/plugins/gitlab/tasks/tag_collector.go
+++ b/backend/plugins/gitlab/tasks/tag_collector.go
@@ -23,6 +23,10 @@ import (
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
+func init() {
+ RegisterSubtaskMeta(&CollectTagMeta)
+}
+
const RAW_TAG_TABLE = "gitlab_api_tag"
var CollectTagMeta = plugin.SubTaskMeta{
@@ -31,6 +35,7 @@ var CollectTagMeta = plugin.SubTaskMeta{
EnabledByDefault: false,
Description: "Collect tag data from gitlab api, does not support
either timeFilter or diffSync.",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
+ Dependencies:
[]*plugin.SubTaskMeta{&ExtractApiMergeRequestDetailsMeta},
}
func CollectApiTag(taskCtx plugin.SubTaskContext) errors.Error {
diff --git a/backend/plugins/gitlab/tasks/tag_extractor.go
b/backend/plugins/gitlab/tasks/tag_extractor.go
index f5c6a258b..9a94de337 100644
--- a/backend/plugins/gitlab/tasks/tag_extractor.go
+++ b/backend/plugins/gitlab/tasks/tag_extractor.go
@@ -25,6 +25,10 @@ import (
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
+func init() {
+ RegisterSubtaskMeta(&ExtractTagMeta)
+}
+
type GitlabApiTag struct {
Name string
Message string
@@ -42,6 +46,7 @@ var ExtractTagMeta = plugin.SubTaskMeta{
EnabledByDefault: false,
Description: "Extract raw tag data into tool layer table
GitlabTag",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
+ Dependencies: []*plugin.SubTaskMeta{&CollectTagMeta},
}
func ExtractApiTag(taskCtx plugin.SubTaskContext) errors.Error {
