This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new e121d8bdf feat(gitlab) fetch trigger jobs (#6167)
e121d8bdf is described below
commit e121d8bdf8183951bf658fb000ef3cf1b2ff9623
Author: Marco-De-Stefani <[email protected]>
AuthorDate: Thu Jan 25 09:55:32 2024 +0100
feat(gitlab) fetch trigger jobs (#6167)
* feat(gitlab): fetching the "trigger" job as well
* feat(gitlab): add test for parsing trigger job
---
.../e2e/raw_tables/_raw_gitlab_api_trigger_job.csv | 2 +
.../_tool_gitlab_jobs_with_triggered.csv | 2 +
backend/plugins/gitlab/e2e/trigger_job_test.go | 73 ++++++++++++++
.../plugins/gitlab/tasks/trigger_job_collector.go | 92 ++++++++++++++++++
.../plugins/gitlab/tasks/trigger_job_extractor.go | 106 +++++++++++++++++++++
5 files changed, 275 insertions(+)
diff --git
a/backend/plugins/gitlab/e2e/raw_tables/_raw_gitlab_api_trigger_job.csv
b/backend/plugins/gitlab/e2e/raw_tables/_raw_gitlab_api_trigger_job.csv
new file mode 100644
index 000000000..d4ae96eea
--- /dev/null
+++ b/backend/plugins/gitlab/e2e/raw_tables/_raw_gitlab_api_trigger_job.csv
@@ -0,0 +1,2 @@
+id,params,data,url,input,created_at
+1,"{""ConnectionId"":1,""ProjectId"":44}","{""id"":1,""status"":""success"",""stage"":""deploy"",""name"":""deploy"",""ref"":""master"",""tag"":false,""coverage"":null,""allow_failure"":false,""created_at"":""2022-08-25T21:19:10.938Z"",""started_at"":null,""finished_at"":""2022-08-25T23:00:08.594Z"",""duration"":null,""user"":{""id"":39,""name"":""
chinese merico chinese
"",""username"":""merico"",""state"":""active"",""avatar_url"":""https://secure.gravatar.com/avatar/8d7684192b6b64e6a
[...]
\ No newline at end of file
diff --git
a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_jobs_with_triggered.csv
b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_jobs_with_triggered.csv
new file mode 100644
index 000000000..70d257432
--- /dev/null
+++
b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_jobs_with_triggered.csv
@@ -0,0 +1,2 @@
+connection_id,gitlab_id,project_id,pipeline_id,status,stage,name,ref,tag,allow_failure,duration,web_url,gitlab_created_at,started_at,finished_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,1,44,73,success,deploy,deploy,master,0,0,0,"",2022-08-25T21:19:10.938+00:00,"",2022-08-25T23:00:08.594+00:00,"{""ConnectionId"":1,""ProjectId"":44}",_raw_gitlab_api_trigger_job,1,
diff --git a/backend/plugins/gitlab/e2e/trigger_job_test.go
b/backend/plugins/gitlab/e2e/trigger_job_test.go
new file mode 100644
index 000000000..4d697dde4
--- /dev/null
+++ b/backend/plugins/gitlab/e2e/trigger_job_test.go
@@ -0,0 +1,73 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "testing"
+
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/gitlab/impl"
+ "github.com/apache/incubator-devlake/plugins/gitlab/models"
+ "github.com/apache/incubator-devlake/plugins/gitlab/tasks"
+)
+
+func TestGitlabTriggerJobDataFlow(t *testing.T) {
+
+ var gitlab impl.Gitlab
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "gitlab", gitlab)
+ regexEnricher := api.NewRegexEnricher()
+ _ = regexEnricher.TryAdd(devops.DEPLOYMENT, "(?i)compile")
+ taskData := &tasks.GitlabTaskData{
+ Options: &tasks.GitlabOptions{
+ ConnectionId: 1,
+ ProjectId: 44,
+ },
+ RegexEnricher: regexEnricher,
+ }
+ // import raw data table
+ // SELECT * FROM _raw_gitlab_api_job INTO OUTFILE
"/tmp/_raw_gitlab_api_job.csv" FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY
'"' LINES TERMINATED BY '\r\n';
+
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_gitlab_api_trigger_job.csv",
"_raw_gitlab_api_trigger_job")
+
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_gitlab_api_job.csv",
"_raw_gitlab_api_job")
+
+ // verify extraction
+ dataflowTester.FlushTabler(&models.GitlabJob{})
+ dataflowTester.Subtask(tasks.ExtractApiTriggerJobsMeta, taskData)
+ dataflowTester.VerifyTable(
+ models.GitlabJob{},
+ "./snapshot_tables/_tool_gitlab_jobs_with_triggered.csv",
+ e2ehelper.ColumnWithRawData(
+ "connection_id",
+ "gitlab_id",
+ "project_id",
+ "pipeline_id",
+ "status",
+ "stage",
+ "name",
+ "ref",
+ "tag",
+ "allow_failure",
+ "duration",
+ "web_url",
+ "gitlab_created_at",
+ "started_at",
+ "finished_at",
+ ),
+ )
+}
diff --git a/backend/plugins/gitlab/tasks/trigger_job_collector.go
b/backend/plugins/gitlab/tasks/trigger_job_collector.go
new file mode 100644
index 000000000..210f0f046
--- /dev/null
+++ b/backend/plugins/gitlab/tasks/trigger_job_collector.go
@@ -0,0 +1,92 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "reflect"
+ "time"
+)
+
+func init() {
+ RegisterSubtaskMeta(&CollectApiTriggerJobsMeta)
+}
+
+const RAW_TRIGGER_JOB_TABLE = "gitlab_api_trigger_job"
+
+var CollectApiTriggerJobsMeta = plugin.SubTaskMeta{
+ Name: "collectApiTriggerJobs",
+ EntryPoint: CollectApiTriggerJobs,
+ EnabledByDefault: true,
+ Description: "Collect job data from gitlab api, supports both
timeFilter and diffSync.",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+ Dependencies: []*plugin.SubTaskMeta{&ExtractApiPipelineDetailsMeta},
+}
+
+func CollectApiTriggerJobs(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_TRIGGER_JOB_TABLE)
+ collectorWithState, err :=
helper.NewStatefulApiCollector(*rawDataSubTaskArgs, data.TimeAfter)
+ tickInterval, err := helper.CalcTickInterval(200, 1*time.Minute)
+ incremental := collectorWithState.IsIncremental()
+
+ iterator, err := GetAllPipelinesIterator(taskCtx, collectorWithState)
+
+ err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
+ //RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ MinTickInterval: &tickInterval,
+ PageSize: 100,
+ Incremental: incremental,
+ Input: iterator,
+ UrlTemplate: "projects/{{ .Params.ProjectId }}/pipelines/{{
.Input.GitlabId }}/bridges",
+ ResponseParser: GetRawMessageFromResponse,
+ AfterResponse: ignoreHTTPStatus403, // ignore 403 for CI/CD
disable
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return collectorWithState.Execute()
+}
+
+func GetAllPipelinesIterator(taskCtx plugin.SubTaskContext, collectorWithState
*helper.ApiCollectorStateManager) (*helper.DalCursorIterator, errors.Error) {
+ db := taskCtx.GetDal()
+ data := taskCtx.GetData().(*GitlabTaskData)
+ clauses := []dal.Clause{
+ dal.Select("gp.gitlab_id, gp.gitlab_id as iid"),
+ dal.From("_tool_gitlab_pipelines gp"),
+ dal.Where(
+ `gp.project_id = ? and gp.connection_id = ? and
gp.gitlab_id not in (select json_extract(tj.input, '$.GitlabId') as gitlab_id
from _raw_gitlab_api_trigger_job tj)`,
+ data.Options.ProjectId, data.Options.ConnectionId,
+ ),
+ }
+ if collectorWithState.LatestState.LatestSuccessStart != nil {
+ clauses = append(clauses, dal.Where("gitlab_updated_at > ?",
*collectorWithState.LatestState.LatestSuccessStart))
+ }
+ // construct the input iterator
+ cursor, err := db.Cursor(clauses...)
+ if err != nil {
+ return nil, err
+ }
+
+ return helper.NewDalCursorIterator(db, cursor,
reflect.TypeOf(GitlabInput{}))
+}
diff --git a/backend/plugins/gitlab/tasks/trigger_job_extractor.go
b/backend/plugins/gitlab/tasks/trigger_job_extractor.go
new file mode 100644
index 000000000..f9b64eeb4
--- /dev/null
+++ b/backend/plugins/gitlab/tasks/trigger_job_extractor.go
@@ -0,0 +1,106 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/gitlab/models"
+ "time"
+)
+
+func init() {
+ RegisterSubtaskMeta(&ExtractApiTriggerJobsMeta)
+}
+
+type ApiTriggerJob struct {
+ Id int `json:"id"`
+ Status string
+ Stage string
+ Name string
+ Ref string
+ Duration float64
+ Pipeline struct {
+ Id int
+ }
+ CreatedAt *time.Time `json:"created_at"`
+ StartedAt *time.Time `json:"started_at"`
+ FinishedAt *time.Time `json:"finished_at"`
+}
+
+var ExtractApiTriggerJobsMeta = plugin.SubTaskMeta{
+ Name: "extractApiTriggerJobs",
+ EntryPoint: ExtractApiTriggerJobs,
+ EnabledByDefault: true,
+ Description: "Extract raw Gitlab trigger jobs data into tool layer
table GitlabPipeline",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+ Dependencies: []*plugin.SubTaskMeta{&CollectApiTriggerJobsMeta},
+}
+
+func ExtractApiTriggerJobs(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_TRIGGER_JOB_TABLE)
+
+ extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
+ // create gitlab commit
+ gitlabApiTriggerJob := &ApiTriggerJob{}
+ err := errors.Convert(json.Unmarshal(row.Data,
gitlabApiTriggerJob))
+ if err != nil {
+ return nil, err
+ }
+
+ gitlabPipeline, err :=
convertTriggerJob(gitlabApiTriggerJob, data.Options.ProjectId)
+ if err != nil {
+ return nil, err
+ }
+
+ // use data.Options.ProjectId to set the value of
ProjectId for it
+ gitlabPipeline.ProjectId = data.Options.ProjectId
+ gitlabPipeline.ConnectionId = data.Options.ConnectionId
+ results := make([]interface{}, 0, 1)
+ results = append(results, gitlabPipeline)
+
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+
+func convertTriggerJob(job *ApiTriggerJob, projectId int) (*models.GitlabJob,
errors.Error) {
+ return &models.GitlabJob{
+ GitlabId: job.Id,
+ ProjectId: projectId,
+ Status: job.Status,
+ Stage: job.Stage,
+ Name: job.Name,
+ Ref: job.Ref,
+ Duration: job.Duration,
+ PipelineId: job.Pipeline.Id,
+ GitlabCreatedAt: job.CreatedAt,
+ StartedAt: job.StartedAt,
+ FinishedAt: job.FinishedAt,
+ }, nil
+}
