This is an automated email from the ASF dual-hosted git repository. warren pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit c45c2b4e2f15b54604646c0728b04f333bac3521 Author: Nddtfjiang <[email protected]> AuthorDate: Fri Jul 29 14:48:08 2022 +0000 feat: gitlab-ci Add CollectApiJobs Add ExtractApiJobs Alert _tool_gitlab_pipelines Nddtfjiang <[email protected]> --- plugins/gitee/api/connection.go | 3 - plugins/gitlab/e2e/pipelines_test.go | 4 +- .../e2e/snapshot_tables/_tool_gitlab_pipelines.csv | 42 ++++---- plugins/gitlab/impl/impl.go | 2 + plugins/gitlab/models/{pipeline.go => job.go} | 35 +++--- .../migrationscripts/20220729_pipeline_and_job.go | 119 +++++++++++++++++++++ plugins/gitlab/models/migrationscripts/register.go | 1 + plugins/gitlab/models/pipeline.go | 28 ++--- plugins/gitlab/tasks/job_collector.go | 54 ++++++++++ plugins/gitlab/tasks/job_extractor.go | 104 ++++++++++++++++++ plugins/gitlab/tasks/pipeline_extractor.go | 73 +++++++------ 11 files changed, 378 insertions(+), 87 deletions(-) diff --git a/plugins/gitee/api/connection.go b/plugins/gitee/api/connection.go index 5fd6e5e7..a0052d38 100644 --- a/plugins/gitee/api/connection.go +++ b/plugins/gitee/api/connection.go @@ -133,8 +133,5 @@ GET /plugins/gitee/connections/:connectionId func GetConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { connection := &models.GiteeConnection{} err := connectionHelper.First(connection, input.Params) - if err != nil { - return nil, err - } return &core.ApiResourceOutput{Body: connection}, err } diff --git a/plugins/gitlab/e2e/pipelines_test.go b/plugins/gitlab/e2e/pipelines_test.go index 06da4ace..ec571393 100644 --- a/plugins/gitlab/e2e/pipelines_test.go +++ b/plugins/gitlab/e2e/pipelines_test.go @@ -58,8 +58,8 @@ func TestGitlabPipelineDataFlow(t *testing.T) { "sha", "web_url", "duration", - "started_at", - "finished_at", + "gitlab_started_at", + "gitlab_finished_at", "coverage", "_raw_data_params", "_raw_data_table", diff --git a/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv b/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv index 73b4672c..46a0333f 100644 --- a/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv +++ b/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_pipelines.csv @@ -1,21 +1,21 @@ -connection_id,gitlab_id,project_id,gitlab_created_at,status,ref,sha,web_url,duration,started_at,finished_at,coverage,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark -1,457474837,12345678,2022-01-27T10:07:09.429+00:00,failed,renovate/pin-dependencies,b1b82852d48b516a18e56c5bab0ebf54b8f4ccfd,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474837,0,2022-01-27T10:07:09.429+00:00,2022-01-27T10:07:09.553+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,152, -1,457474996,12345678,2022-01-27T10:07:18.884+00:00,failed,renovate/jest-monorepo,739ab912e6e1ed27cecd8a2d00bfd6fa52afd90a,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474996,0,2022-01-27T10:07:18.884+00:00,2022-01-27T10:07:19.043+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,153, -1,457475160,12345678,2022-01-27T10:07:26.435+00:00,failed,renovate/lodash-monorepo,44d127e0ab7dbc4bc259b55929c9d00b62fc3bf4,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475160,0,2022-01-27T10:07:26.435+00:00,2022-01-27T10:07:26.638+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,154, -1,457475337,12345678,2022-01-27T10:07:36.502+00:00,failed,renovate/shx-0.x,10a6464b6bd2cf4b59b8ac37ce1466e013f5a20d,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475337,1,2022-01-27T10:07:36.502+00:00,2022-01-27T10:07:37.545+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,155, -1,485811050,12345678,2022-03-07T06:26:42.109+00:00,failed,EE-7121,c791ea6949d6b4aadf79b15ba666cb690c6527ac,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811050,0,2022-03-07T06:26:42.109+00:00,2022-03-07T06:26:42.109+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,156, -1,485811059,12345678,2022-03-07T06:26:43.784+00:00,failed,refs/merge-requests/74/head,c791ea6949d6b4aadf79b15ba666cb690c6527ac,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811059,0,2022-03-07T06:26:43.784+00:00,2022-03-07T06:26:43.784+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,157, -1,485813816,12345678,2022-03-07T06:33:56.824+00:00,failed,EE-7121,ecc7c0b2874c812ed882c9effbbda26e0abc7110,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813816,0,2022-03-07T06:33:56.824+00:00,2022-03-07T06:33:56.824+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,158, -1,485813830,12345678,2022-03-07T06:33:58.889+00:00,failed,refs/merge-requests/74/head,ecc7c0b2874c812ed882c9effbbda26e0abc7110,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813830,0,2022-03-07T06:33:58.889+00:00,2022-03-07T06:33:58.889+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,159, -1,485814501,12345678,2022-03-07T06:35:28.111+00:00,failed,EE-7121,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814501,0,2022-03-07T06:35:28.111+00:00,2022-03-07T06:35:28.111+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,160, -1,485814516,12345678,2022-03-07T06:35:31.255+00:00,failed,refs/merge-requests/74/head,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814516,0,2022-03-07T06:35:31.255+00:00,2022-03-07T06:35:31.255+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,161, -1,485814871,12345678,2022-03-07T06:36:50.020+00:00,failed,refs/merge-requests/74/head,f731732471961fed061a01fa99631f69e861c4a2,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814871,42,2022-03-07T06:36:50.020+00:00,2022-03-07T06:37:32.103+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,162, -1,485817670,12345678,2022-03-07T06:45:09.471+00:00,failed,refs/merge-requests/74/head,5b95c5aebce1eae6a1b95ecf6fbc870851455375,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485817670,1956,2022-03-07T06:45:09.471+00:00,2022-03-07T07:17:46.305+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,163, -1,485837602,12345678,2022-03-07T07:20:45.859+00:00,failed,refs/merge-requests/74/head,a79e2aaeee9916338a11b665cb4201f24627f3f1,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485837602,434,2022-03-07T07:20:45.859+00:00,2022-03-07T07:28:00.277+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,164, -1,485842553,12345678,2022-03-07T07:30:47.018+00:00,failed,refs/merge-requests/74/head,9e0934cedcd83abab8d34c5bb0f597c7837a55f0,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485842553,287,2022-03-07T07:30:47.018+00:00,2022-03-07T07:35:34.998+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,165, -1,485845850,12345678,2022-03-07T07:38:58.611+00:00,failed,refs/merge-requests/74/head,b3b581940f6d4d3226db31e48eab2b3b73f4c6db,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485845850,419,2022-03-07T07:38:58.611+00:00,2022-03-07T07:45:58.412+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,166, -1,485852752,12345678,2022-03-07T07:46:09.385+00:00,failed,refs/merge-requests/74/head,82df93eeb72c71271dad5a85359f39661de899fe,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485852752,319,2022-03-07T07:46:09.385+00:00,2022-03-07T07:51:28.709+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,167, -1,485865876,12345678,2022-03-07T08:04:56.406+00:00,failed,refs/merge-requests/74/head,db45efd231dbaca1337bd29bc1f65f404754e6bd,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485865876,480,2022-03-07T08:04:56.406+00:00,2022-03-07T08:12:56.453+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,168, -1,485877118,12345678,2022-03-07T08:22:48.943+00:00,failed,refs/merge-requests/74/head,09f81b1b2d083411c0bfecd32d7728479b594503,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485877118,289,2022-03-07T08:22:48.943+00:00,2022-03-07T08:27:38.364+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,169, -1,485905167,12345678,2022-03-07T09:02:09.994+00:00,failed,refs/merge-requests/74/head,cce432655861efafcd05edc8de09e305c772d2b8,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485905167,687,2022-03-07T09:02:09.994+00:00,2022-03-07T09:13:37.013+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,170, -1,485932863,12345678,2022-03-07T09:34:57.476+00:00,success,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb279,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,2022-03-07T09:34:57.476+00:00,2022-03-07T09:41:36.267+00:00,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,171, +connection_id,gitlab_id,project_id,gitlab_created_at,status,ref,sha,web_url,duration,gitlab_started_at,gitlab_finished_at,coverage,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +1,457474837,12345678,2022-01-27T10:07:09.429+00:00,failed,renovate/pin-dependencies,b1b82852d48b516a18e56c5bab0ebf54b8f4ccfd,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474837,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,152, +1,457474996,12345678,2022-01-27T10:07:18.884+00:00,failed,renovate/jest-monorepo,739ab912e6e1ed27cecd8a2d00bfd6fa52afd90a,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457474996,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,153, +1,457475160,12345678,2022-01-27T10:07:26.435+00:00,failed,renovate/lodash-monorepo,44d127e0ab7dbc4bc259b55929c9d00b62fc3bf4,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475160,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,154, +1,457475337,12345678,2022-01-27T10:07:36.502+00:00,failed,renovate/shx-0.x,10a6464b6bd2cf4b59b8ac37ce1466e013f5a20d,https://gitlab.com/merico-dev/ee/charts/-/pipelines/457475337,1,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,155, +1,485811050,12345678,2022-03-07T06:26:42.109+00:00,failed,EE-7121,c791ea6949d6b4aadf79b15ba666cb690c6527ac,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811050,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,156, +1,485811059,12345678,2022-03-07T06:26:43.784+00:00,failed,refs/merge-requests/74/head,c791ea6949d6b4aadf79b15ba666cb690c6527ac,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485811059,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,157, +1,485813816,12345678,2022-03-07T06:33:56.824+00:00,failed,EE-7121,ecc7c0b2874c812ed882c9effbbda26e0abc7110,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813816,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,158, +1,485813830,12345678,2022-03-07T06:33:58.889+00:00,failed,refs/merge-requests/74/head,ecc7c0b2874c812ed882c9effbbda26e0abc7110,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485813830,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,159, +1,485814501,12345678,2022-03-07T06:35:28.111+00:00,failed,EE-7121,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814501,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,160, +1,485814516,12345678,2022-03-07T06:35:31.255+00:00,failed,refs/merge-requests/74/head,6a3346f8434cc65fbe3f7a80a0edec5b4014a733,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814516,0,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,161, +1,485814871,12345678,2022-03-07T06:36:50.020+00:00,failed,refs/merge-requests/74/head,f731732471961fed061a01fa99631f69e861c4a2,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485814871,42,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,162, +1,485817670,12345678,2022-03-07T06:45:09.471+00:00,failed,refs/merge-requests/74/head,5b95c5aebce1eae6a1b95ecf6fbc870851455375,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485817670,1956,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,163, +1,485837602,12345678,2022-03-07T07:20:45.859+00:00,failed,refs/merge-requests/74/head,a79e2aaeee9916338a11b665cb4201f24627f3f1,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485837602,434,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,164, +1,485842553,12345678,2022-03-07T07:30:47.018+00:00,failed,refs/merge-requests/74/head,9e0934cedcd83abab8d34c5bb0f597c7837a55f0,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485842553,287,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,165, +1,485845850,12345678,2022-03-07T07:38:58.611+00:00,failed,refs/merge-requests/74/head,b3b581940f6d4d3226db31e48eab2b3b73f4c6db,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485845850,419,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,166, +1,485852752,12345678,2022-03-07T07:46:09.385+00:00,failed,refs/merge-requests/74/head,82df93eeb72c71271dad5a85359f39661de899fe,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485852752,319,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,167, +1,485865876,12345678,2022-03-07T08:04:56.406+00:00,failed,refs/merge-requests/74/head,db45efd231dbaca1337bd29bc1f65f404754e6bd,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485865876,480,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,168, +1,485877118,12345678,2022-03-07T08:22:48.943+00:00,failed,refs/merge-requests/74/head,09f81b1b2d083411c0bfecd32d7728479b594503,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485877118,289,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,169, +1,485905167,12345678,2022-03-07T09:02:09.994+00:00,failed,refs/merge-requests/74/head,cce432655861efafcd05edc8de09e305c772d2b8,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485905167,687,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,170, +1,485932863,12345678,2022-03-07T09:34:57.476+00:00,success,refs/merge-requests/74/head,12fc3a42080bb98ca520817bd4fe0ca33c0bb279,https://gitlab.com/merico-dev/ee/charts/-/pipelines/485932863,398,,,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_pipeline,171, diff --git a/plugins/gitlab/impl/impl.go b/plugins/gitlab/impl/impl.go index 4c62da6f..8b2401a1 100644 --- a/plugins/gitlab/impl/impl.go +++ b/plugins/gitlab/impl/impl.go @@ -64,6 +64,8 @@ func (plugin Gitlab) SubTaskMetas() []core.SubTaskMeta { tasks.ExtractApiMrCommitsMeta, tasks.CollectApiPipelinesMeta, tasks.ExtractApiPipelinesMeta, + tasks.CollectApiJobsMeta, + tasks.ExtractApiJobsMeta, tasks.EnrichMergeRequestsMeta, tasks.CollectAccountsMeta, tasks.ExtractAccountsMeta, diff --git a/plugins/gitlab/models/pipeline.go b/plugins/gitlab/models/job.go similarity index 59% copy from plugins/gitlab/models/pipeline.go copy to plugins/gitlab/models/job.go index 2c16a74c..9e437a9a 100644 --- a/plugins/gitlab/models/pipeline.go +++ b/plugins/gitlab/models/job.go @@ -18,27 +18,32 @@ limitations under the License. package models import ( - "github.com/apache/incubator-devlake/models/common" "time" + + "github.com/apache/incubator-devlake/models/common" ) -type GitlabPipeline struct { +type GitlabJob struct { ConnectionId uint64 `gorm:"primaryKey"` - GitlabId int `gorm:"primaryKey"` - ProjectId int `gorm:"index"` - GitlabCreatedAt time.Time - Status string `gorm:"type:varchar(100)"` - Ref string `gorm:"type:varchar(255)"` - Sha string `gorm:"type:varchar(255)"` - WebUrl string `gorm:"type:varchar(255)"` - Duration int - StartedAt *time.Time - FinishedAt *time.Time - Coverage string + GitlabId int `gorm:"primaryKey"` + ProjectId int `gorm:"index"` + Status string `gorm:"type:varchar(255)"` + Stage string `gorm:"type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Ref string `gorm:"type:varchar(255)"` + Tag bool `gorm:"type:boolean"` + AllowFailure bool `json:"allow_failure"` + Duration float64 `gorm:"type:text"` + WebUrl string `gorm:"type:varchar(255)"` + + GitlabCreatedAt *time.Time + GitlabStartedAt *time.Time + GitlabFinishedAt *time.Time + common.NoPKModel } -func (GitlabPipeline) TableName() string { - return "_tool_gitlab_pipelines" +func (GitlabJob) TableName() string { + return "_tool_gitlab_jobs" } diff --git a/plugins/gitlab/models/migrationscripts/20220729_pipeline_and_job.go b/plugins/gitlab/models/migrationscripts/20220729_pipeline_and_job.go new file mode 100644 index 00000000..7a5a384f --- /dev/null +++ b/plugins/gitlab/models/migrationscripts/20220729_pipeline_and_job.go @@ -0,0 +1,119 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "context" + "time" + + "github.com/apache/incubator-devlake/models/common" + "github.com/apache/incubator-devlake/plugins/gitlab/models/migrationscripts/archived" + "gorm.io/gorm" +) + +type alertPipeline struct{} + +type GitlabPipeline20220729 struct { + ConnectionId uint64 `gorm:"primaryKey"` + + GitlabId int `gorm:"primaryKey"` + ProjectId int `gorm:"index"` + Status string `gorm:"type:varchar(100)"` + Ref string `gorm:"type:varchar(255)"` + Sha string `gorm:"type:varchar(255)"` + WebUrl string `gorm:"type:varchar(255)"` + Duration int + + GitlabCreatedAt *time.Time + GitlabUpdatedAt *time.Time + GitlabStartedAt *time.Time + GitlabFinishedAt *time.Time + Coverage string + + common.NoPKModel +} + +func (GitlabPipeline20220729) TableName() string { + return "_tool_gitlab_pipelines" +} + +type GitlabJob20220729 struct { + ConnectionId uint64 `gorm:"primaryKey"` + + GitlabId int `gorm:"primaryKey"` + ProjectId int `gorm:"index"` + Status string `gorm:"type:varchar(255)"` + Stage string `gorm:"type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Ref string `gorm:"type:varchar(255)"` + Tag bool `gorm:"type:boolean"` + AllowFailure bool `json:"allow_failure"` + Duration float64 `gorm:"type:text"` + WebUrl string `gorm:"type:varchar(255)"` + + GitlabCreatedAt *time.Time + GitlabStartedAt *time.Time + GitlabFinishedAt *time.Time + + common.NoPKModel +} + +func (GitlabJob20220729) TableName() string { + return "_tool_gitlab_jobs" +} + +func (*alertPipeline) Up(ctx context.Context, db *gorm.DB) error { + err := db.Migrator().DropColumn(&archived.GitlabPipeline{}, "started_at") + if err != nil { + return err + } + err = db.Migrator().DropColumn(&archived.GitlabPipeline{}, "finished_at") + if err != nil { + return err + } + + err = db.Migrator().AddColumn(&GitlabPipeline20220729{}, "gitlab_updated_at") + if err != nil { + return err + } + + err = db.Migrator().AddColumn(&GitlabPipeline20220729{}, "gitlab_started_at") + if err != nil { + return err + } + + err = db.Migrator().AddColumn(&GitlabPipeline20220729{}, "gitlab_finished_at") + if err != nil { + return err + } + + err = db.Migrator().AutoMigrate(&GitlabJob20220729{}) + if err != nil { + return err + } + + return nil +} + +func (*alertPipeline) Version() uint64 { + return 20220729231236 +} + +func (*alertPipeline) Name() string { + return "pipeline and job" +} diff --git a/plugins/gitlab/models/migrationscripts/register.go b/plugins/gitlab/models/migrationscripts/register.go index c1365f7d..b47b0d6f 100644 --- a/plugins/gitlab/models/migrationscripts/register.go +++ b/plugins/gitlab/models/migrationscripts/register.go @@ -25,5 +25,6 @@ import ( func All() []migration.Script { return []migration.Script{ new(addInitTables), + new(alertPipeline), } } diff --git a/plugins/gitlab/models/pipeline.go b/plugins/gitlab/models/pipeline.go index 2c16a74c..6f9d3532 100644 --- a/plugins/gitlab/models/pipeline.go +++ b/plugins/gitlab/models/pipeline.go @@ -18,24 +18,28 @@ limitations under the License. package models import ( - "github.com/apache/incubator-devlake/models/common" "time" + + "github.com/apache/incubator-devlake/models/common" ) type GitlabPipeline struct { ConnectionId uint64 `gorm:"primaryKey"` - GitlabId int `gorm:"primaryKey"` - ProjectId int `gorm:"index"` - GitlabCreatedAt time.Time - Status string `gorm:"type:varchar(100)"` - Ref string `gorm:"type:varchar(255)"` - Sha string `gorm:"type:varchar(255)"` - WebUrl string `gorm:"type:varchar(255)"` - Duration int - StartedAt *time.Time - FinishedAt *time.Time - Coverage string + GitlabId int `gorm:"primaryKey"` + ProjectId int `gorm:"index"` + Status string `gorm:"type:varchar(100)"` + Ref string `gorm:"type:varchar(255)"` + Sha string `gorm:"type:varchar(255)"` + WebUrl string `gorm:"type:varchar(255)"` + Duration int + + GitlabCreatedAt *time.Time + GitlabUpdatedAt *time.Time + GitlabStartedAt *time.Time + GitlabFinishedAt *time.Time + Coverage string + common.NoPKModel } diff --git a/plugins/gitlab/tasks/job_collector.go b/plugins/gitlab/tasks/job_collector.go new file mode 100644 index 00000000..a8a26c42 --- /dev/null +++ b/plugins/gitlab/tasks/job_collector.go @@ -0,0 +1,54 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/helper" +) + +const RAW_JOB_TABLE = "gitlab_api_job" + +var CollectApiJobsMeta = core.SubTaskMeta{ + Name: "collectApiJobs", + EntryPoint: CollectApiJobs, + EnabledByDefault: true, + Description: "Collect job data from gitlab api", + DomainTypes: []string{core.DOMAIN_TYPE_CICD}, +} + +func CollectApiJobs(taskCtx core.SubTaskContext) error { + rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_JOB_TABLE) + + collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{ + RawDataSubTaskArgs: *rawDataSubTaskArgs, + ApiClient: data.ApiClient, + PageSize: 100, + Incremental: false, + UrlTemplate: "projects/{{ .Params.ProjectId }}/jobs", + Query: GetQuery, + ResponseParser: GetRawMessageFromResponse, + AfterResponse: ignoreHTTPStatus403, // ignore 403 for CI/CD disable + }) + + if err != nil { + return err + } + + return collector.Execute() +} diff --git a/plugins/gitlab/tasks/job_extractor.go b/plugins/gitlab/tasks/job_extractor.go new file mode 100644 index 00000000..d7d99d04 --- /dev/null +++ b/plugins/gitlab/tasks/job_extractor.go @@ -0,0 +1,104 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/gitlab/models" + "github.com/apache/incubator-devlake/plugins/helper" +) + +type ApiJob struct { + Id int `json:"id"` + Status string + Stage string + Name string + Ref string + Tag bool + AllowFailure bool `json:"allow_failure"` + Duration float64 + WebUrl string `json:"web_url"` + + CreatedAt *helper.Iso8601Time `json:"created_at"` + StartedAt *helper.Iso8601Time `json:"started_at"` + FinishedAt *helper.Iso8601Time `json:"finished_at"` +} + +var ExtractApiJobsMeta = core.SubTaskMeta{ + Name: "extractApiJobs", + EntryPoint: ExtractApiJobs, + EnabledByDefault: true, + Description: "Extract raw pipelines data into tool layer table GitlabPipeline", + DomainTypes: []string{core.DOMAIN_TYPE_CICD}, +} + +func ExtractApiJobs(taskCtx core.SubTaskContext) error { + rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_JOB_TABLE) + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: *rawDataSubTaskArgs, + Extract: func(row *helper.RawData) ([]interface{}, error) { + // create gitlab commit + gitlabApiJob := &ApiJob{} + err := json.Unmarshal(row.Data, gitlabApiJob) + if err != nil { + return nil, err + } + + gitlabPipeline, err := convertJob(gitlabApiJob, data.Options.ProjectId) + if err != nil { + return nil, err + } + + // use data.Options.ProjectId to set the value of ProjectId for it + gitlabPipeline.ProjectId = data.Options.ProjectId + gitlabPipeline.ConnectionId = data.Options.ConnectionId + results := make([]interface{}, 0, 1) + results = append(results, gitlabPipeline) + + return results, nil + }, + }) + + if err != nil { + return err + } + + return extractor.Execute() +} + +func convertJob(job *ApiJob, projectId int) (*models.GitlabJob, error) { + return &models.GitlabJob{ + GitlabId: job.Id, + ProjectId: projectId, + Status: job.Status, + Stage: job.Stage, + Name: job.Name, + Ref: job.Ref, + Tag: job.Tag, + AllowFailure: job.AllowFailure, + Duration: job.Duration, + WebUrl: job.WebUrl, + + GitlabCreatedAt: helper.Iso8601TimeToTime(job.CreatedAt), + GitlabStartedAt: helper.Iso8601TimeToTime(job.StartedAt), + GitlabFinishedAt: helper.Iso8601TimeToTime(job.FinishedAt), + }, nil +} diff --git a/plugins/gitlab/tasks/pipeline_extractor.go b/plugins/gitlab/tasks/pipeline_extractor.go index 74a57cd0..25885b5f 100644 --- a/plugins/gitlab/tasks/pipeline_extractor.go +++ b/plugins/gitlab/tasks/pipeline_extractor.go @@ -25,29 +25,32 @@ import ( "github.com/apache/incubator-devlake/plugins/helper" ) -type ApiPipeline struct { - GitlabId int `json:"id"` - ProjectId int `json:"project_id"` - GitlabCreatedAt *helper.Iso8601Time `json:"created_at"` - UpdatedAt *helper.Iso8601Time `json:"updated_at"` - Ref string - Sha string - Duration int - WebUrl string `json:"web_url"` - Status string +type ApiDetailedStatus struct { + Icon string + Text string + Label string + Group string + Tooltip string + HasDetails bool `json:"has_details"` + DetailsPath string `json:"details_path"` + Favicon string } -type ApiSinglePipelineResponse struct { - GitlabId int `json:"id"` - ProjectId int `json:"project_id"` - GitlabCreatedAt *helper.Iso8601Time `json:"created_at"` - Ref string - Sha string - WebUrl string `json:"web_url"` - Duration int - UpdatedAt *helper.Iso8601Time `json:"updated_at"` - Coverage string - Status string +type ApiPipeline struct { + Id int `json:"id"` + Ref string + Sha string + Status string + Tag bool + Duration int + WebUrl string `json:"web_url"` + + CreatedAt *helper.Iso8601Time `json:"created_at"` + UpdatedAt *helper.Iso8601Time `json:"updated_at"` + StartedAt *helper.Iso8601Time `json:"started_at"` + FinishedAt *helper.Iso8601Time `json:"finished_at"` + + ApiDetailedStatus } var ExtractApiPipelinesMeta = core.SubTaskMeta{ @@ -70,9 +73,10 @@ func ExtractApiPipelines(taskCtx core.SubTaskContext) error { if err != nil { return nil, err } - duration := int(gitlabApiPipeline.UpdatedAt.ToTime().Sub(gitlabApiPipeline.GitlabCreatedAt.ToTime()).Seconds()) + + duration := int(gitlabApiPipeline.UpdatedAt.ToTime().Sub(gitlabApiPipeline.CreatedAt.ToTime()).Seconds()) gitlabApiPipeline.Duration = duration - gitlabPipeline, err := convertPipeline(gitlabApiPipeline) + gitlabPipeline, err := convertPipeline(gitlabApiPipeline, data.Options.ProjectId) if err != nil { return nil, err } @@ -94,18 +98,19 @@ func ExtractApiPipelines(taskCtx core.SubTaskContext) error { return extractor.Execute() } -func convertPipeline(pipeline *ApiPipeline) (*models.GitlabPipeline, error) { +func convertPipeline(pipeline *ApiPipeline, projectId int) (*models.GitlabPipeline, error) { gitlabPipeline := &models.GitlabPipeline{ - GitlabId: pipeline.GitlabId, - ProjectId: pipeline.ProjectId, - GitlabCreatedAt: pipeline.GitlabCreatedAt.ToTime(), - Ref: pipeline.Ref, - Sha: pipeline.Sha, - WebUrl: pipeline.WebUrl, - Status: pipeline.Status, - StartedAt: helper.Iso8601TimeToTime(pipeline.GitlabCreatedAt), - FinishedAt: helper.Iso8601TimeToTime(pipeline.UpdatedAt), - Duration: pipeline.Duration, + GitlabId: pipeline.Id, + ProjectId: projectId, + Ref: pipeline.Ref, + Sha: pipeline.Sha, + WebUrl: pipeline.WebUrl, + Status: pipeline.Status, + GitlabCreatedAt: helper.Iso8601TimeToTime(pipeline.CreatedAt), + GitlabUpdatedAt: helper.Iso8601TimeToTime(pipeline.UpdatedAt), + GitlabStartedAt: helper.Iso8601TimeToTime(pipeline.StartedAt), + GitlabFinishedAt: helper.Iso8601TimeToTime(pipeline.FinishedAt), + Duration: pipeline.Duration, } return gitlabPipeline, nil }
