This is an automated email from the ASF dual-hosted git repository. abeizn pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 059f2a9fae5c30ade034309257695fd0b1ea073e Author: linyh <[email protected]> AuthorDate: Thu Sep 15 17:19:53 2022 +0800 feat: rename cicd_pipeline_repos to cicd_pipeline_commits --- generator/cmd/create_migration.go | 2 +- models/domainlayer/devops/cicd_pipeline_repo.go | 16 +++-- .../20220915_rename_pipeline_commits.go | 69 ++++++++++++++++++++++ models/migrationscripts/register.go | 1 + plugins/github/e2e/cicd_test.go | 7 ++- plugins/github/tasks/cicd_pipeline_convertor.go | 10 ++-- plugins/gitlab/e2e/pipelines_test.go | 8 +-- plugins/gitlab/tasks/pipeline_project_convertor.go | 11 ++-- plugins/jenkins/e2e/builds_test.go | 8 +-- plugins/jenkins/tasks/build_repo_convertor.go | 9 +-- 10 files changed, 102 insertions(+), 39 deletions(-) diff --git a/generator/cmd/create_migration.go b/generator/cmd/create_migration.go index e4c9daff..9a4d9a4f 100644 --- a/generator/cmd/create_migration.go +++ b/generator/cmd/create_migration.go @@ -51,7 +51,7 @@ If framework passed, generator will create a new migration in models/migrationsc pluginName = args[0] } if pluginName == `` { - pluginItems, err := pluginNames(false) + pluginItems, err := pluginNames(true) cobra.CheckErr(err) prompt := promptui.Select{ Label: "plugin_name", diff --git a/models/domainlayer/devops/cicd_pipeline_repo.go b/models/domainlayer/devops/cicd_pipeline_repo.go index ec237182..b19cc636 100644 --- a/models/domainlayer/devops/cicd_pipeline_repo.go +++ b/models/domainlayer/devops/cicd_pipeline_repo.go @@ -17,15 +17,13 @@ limitations under the License. package devops -import "github.com/apache/incubator-devlake/models/domainlayer" - -type CiCDPipelineRepo struct { - domainlayer.DomainEntity - CommitSha string `gorm:"primaryKey;type:varchar(255)"` - Branch string `gorm:"type:varchar(255)"` - Repo string `gorm:"type:varchar(255)"` +type CiCDPipelineCommit struct { + PipelineId string `gorm:"primaryKey;type:varchar(255)"` + CommitSha string `gorm:"primaryKey;type:varchar(255)"` + Branch string `gorm:"type:varchar(255)"` + Repo string `gorm:"type:varchar(255)"` } -func (CiCDPipelineRepo) TableName() string { - return "cicd_pipeline_repos" +func (CiCDPipelineCommit) TableName() string { + return "cicd_pipeline_commits" } diff --git a/models/migrationscripts/20220915_rename_pipeline_commits.go b/models/migrationscripts/20220915_rename_pipeline_commits.go new file mode 100644 index 00000000..0ff50152 --- /dev/null +++ b/models/migrationscripts/20220915_rename_pipeline_commits.go @@ -0,0 +1,69 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "context" + "github.com/apache/incubator-devlake/errors" + "github.com/apache/incubator-devlake/models/domainlayer" + "gorm.io/gorm" +) + +type renamePipelineCommits struct{} + +type CiCDPipelineRepoOld struct { + domainlayer.DomainEntity + CommitSha string `gorm:"primaryKey;type:varchar(255)"` + Branch string `gorm:"type:varchar(255)"` + Repo string `gorm:"type:varchar(255)"` +} + +func (CiCDPipelineRepoOld) TableName() string { + return "cicd_pipeline_repos" +} + +type CiCDPipelineRepo0915 struct { + PipelineId string `gorm:"primaryKey;type:varchar(255)"` + CommitSha string `gorm:"primaryKey;type:varchar(255)"` + Branch string `gorm:"type:varchar(255)"` + Repo string `gorm:"type:varchar(255)"` +} + +func (CiCDPipelineRepo0915) TableName() string { + return "cicd_pipeline_commits" +} + +func (*renamePipelineCommits) Up(ctx context.Context, db *gorm.DB) errors.Error { + err := db.Migrator().RenameTable(CiCDPipelineRepoOld{}, CiCDPipelineRepo0915{}) + if err != nil { + return errors.Convert(err) + } + err = db.Migrator().RenameColumn(CiCDPipelineRepo0915{}, `id`, `pipeline_id`) + if err != nil { + return errors.Convert(err) + } + return nil +} + +func (*renamePipelineCommits) Version() uint64 { + return 20220915000025 +} + +func (*renamePipelineCommits) Name() string { + return "UpdateSchemas for renamePipelineCommits" +} diff --git a/models/migrationscripts/register.go b/models/migrationscripts/register.go index d90dffc5..d9b57dc9 100644 --- a/models/migrationscripts/register.go +++ b/models/migrationscripts/register.go @@ -46,5 +46,6 @@ func All() []migration.Script { new(modifyBoardRepos), new(addOriginChangeValueForPr), new(addCommitFilePathLength), + new(renamePipelineCommits), } } diff --git a/plugins/github/e2e/cicd_test.go b/plugins/github/e2e/cicd_test.go index 5482097d..984c7ebe 100644 --- a/plugins/github/e2e/cicd_test.go +++ b/plugins/github/e2e/cicd_test.go @@ -123,7 +123,7 @@ func TestGithubRunsDataFlow(t *testing.T) { // verify extraction dataflowTester.FlushTabler(&models.GithubRun{}) dataflowTester.FlushTabler(&devops.CICDPipeline{}) - dataflowTester.FlushTabler(&devops.CiCDPipelineRepo{}) + dataflowTester.FlushTabler(&devops.CiCDPipelineCommit{}) dataflowTester.Subtask(tasks.ExtractRunsMeta, taskData) dataflowTester.VerifyTable( @@ -185,9 +185,10 @@ func TestGithubRunsDataFlow(t *testing.T) { ) dataflowTester.VerifyTable( - &devops.CiCDPipelineRepo{}, - "./snapshot_tables/cicd_pipeline_repos.csv", + &devops.CiCDPipelineCommit{}, + "./snapshot_tables/cicd_pipeline_commits.csv", []string{ + "pipeline_id", "commit_sha", "branch", "repo", diff --git a/plugins/github/tasks/cicd_pipeline_convertor.go b/plugins/github/tasks/cicd_pipeline_convertor.go index b701d99f..b67c3d59 100644 --- a/plugins/github/tasks/cicd_pipeline_convertor.go +++ b/plugins/github/tasks/cicd_pipeline_convertor.go @@ -93,11 +93,11 @@ func ConvertPipelines(taskCtx core.SubTaskContext) errors.Error { domainPipeline.DurationSec = uint64(line.GithubUpdatedAt.Sub(*line.GithubCreatedAt).Seconds()) } - domainPipelineProject := &devops.CiCDPipelineRepo{ - DomainEntity: domainlayer.DomainEntity{Id: fmt.Sprintf("%s:%s:%d:%d", "github", "GithubRun", data.Options.ConnectionId, line.ID)}, - CommitSha: line.HeadSha, - Branch: line.HeadBranch, - Repo: didgen.NewDomainIdGenerator(&githubModels.GithubRepo{}).Generate(data.Options.ConnectionId, repoId), + domainPipelineProject := &devops.CiCDPipelineCommit{ + PipelineId: fmt.Sprintf("%s:%s:%d:%d", "github", "GithubRun", data.Options.ConnectionId, line.ID), + CommitSha: line.HeadSha, + Branch: line.HeadBranch, + Repo: didgen.NewDomainIdGenerator(&githubModels.GithubRepo{}).Generate(data.Options.ConnectionId, repoId), } return []interface{}{ diff --git a/plugins/gitlab/e2e/pipelines_test.go b/plugins/gitlab/e2e/pipelines_test.go index e15699a8..1a5c7acf 100644 --- a/plugins/gitlab/e2e/pipelines_test.go +++ b/plugins/gitlab/e2e/pipelines_test.go @@ -83,7 +83,7 @@ func TestGitlabPipelineDataFlow(t *testing.T) { // verify conversion dataflowTester.FlushTabler(&devops.CICDPipeline{}) - dataflowTester.FlushTabler(&devops.CiCDPipelineRepo{}) + dataflowTester.FlushTabler(&devops.CiCDPipelineCommit{}) dataflowTester.Subtask(tasks.ConvertPipelineMeta, taskData) dataflowTester.Subtask(tasks.ConvertPipelineProjectMeta, taskData) dataflowTester.VerifyTable( @@ -103,10 +103,10 @@ func TestGitlabPipelineDataFlow(t *testing.T) { ) dataflowTester.VerifyTable( - devops.CiCDPipelineRepo{}, - "./snapshot_tables/cicd_pipeline_repos.csv", + devops.CiCDPipelineCommit{}, + "./snapshot_tables/cicd_pipeline_commits.csv", []string{ - "id", + "pipeline_id", "commit_sha", "branch", "repo", diff --git a/plugins/gitlab/tasks/pipeline_project_convertor.go b/plugins/gitlab/tasks/pipeline_project_convertor.go index d3c2217a..650b5d05 100644 --- a/plugins/gitlab/tasks/pipeline_project_convertor.go +++ b/plugins/gitlab/tasks/pipeline_project_convertor.go @@ -19,7 +19,6 @@ package tasks import ( "github.com/apache/incubator-devlake/errors" - "github.com/apache/incubator-devlake/models/domainlayer" "github.com/apache/incubator-devlake/models/domainlayer/devops" "github.com/apache/incubator-devlake/models/domainlayer/didgen" "github.com/apache/incubator-devlake/plugins/core" @@ -63,12 +62,10 @@ func ConvertPipelineProjects(taskCtx core.SubTaskContext) errors.Error { Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { gitlabPipelineProject := inputRow.(*gitlabModels.GitlabPipelineProject) - domainPipelineRepo := &devops.CiCDPipelineRepo{ - DomainEntity: domainlayer.DomainEntity{ - Id: pipelineIdGen.Generate(data.Options.ConnectionId, gitlabPipelineProject.PipelineId), - }, - CommitSha: gitlabPipelineProject.Sha, - Branch: gitlabPipelineProject.Ref, + domainPipelineRepo := &devops.CiCDPipelineCommit{ + PipelineId: pipelineIdGen.Generate(data.Options.ConnectionId, gitlabPipelineProject.PipelineId), + CommitSha: gitlabPipelineProject.Sha, + Branch: gitlabPipelineProject.Ref, Repo: didgen.NewDomainIdGenerator(&gitlabModels.GitlabProject{}). Generate(gitlabPipelineProject.ConnectionId, gitlabPipelineProject.ProjectId), } diff --git a/plugins/jenkins/e2e/builds_test.go b/plugins/jenkins/e2e/builds_test.go index 6d67e3f6..0880a36b 100644 --- a/plugins/jenkins/e2e/builds_test.go +++ b/plugins/jenkins/e2e/builds_test.go @@ -87,7 +87,7 @@ func TestJenkinsBuildsDataFlow(t *testing.T) { dataflowTester.FlushTabler(&devops.CICDTask{}) dataflowTester.FlushTabler(&devops.CICDPipeline{}) - dataflowTester.FlushTabler(&devops.CiCDPipelineRepo{}) + dataflowTester.FlushTabler(&devops.CiCDPipelineCommit{}) dataflowTester.FlushTabler(&devops.CICDPipelineRelationship{}) dataflowTester.Subtask(tasks.EnrichApiBuildWithStagesMeta, taskData) dataflowTester.Subtask(tasks.ConvertBuildsToCICDMeta, taskData) @@ -133,10 +133,10 @@ func TestJenkinsBuildsDataFlow(t *testing.T) { ) dataflowTester.VerifyTable( - devops.CiCDPipelineRepo{}, - "./snapshot_tables/cicd_pipeline_repos.csv", + devops.CiCDPipelineCommit{}, + "./snapshot_tables/cicd_pipeline_commits.csv", []string{ - "id", + "pipeline_id", "repo", "branch", "commit_sha", diff --git a/plugins/jenkins/tasks/build_repo_convertor.go b/plugins/jenkins/tasks/build_repo_convertor.go index 76141675..54804337 100644 --- a/plugins/jenkins/tasks/build_repo_convertor.go +++ b/plugins/jenkins/tasks/build_repo_convertor.go @@ -22,7 +22,6 @@ import ( "github.com/apache/incubator-devlake/errors" "reflect" - "github.com/apache/incubator-devlake/models/domainlayer" "github.com/apache/incubator-devlake/models/domainlayer/devops" "github.com/apache/incubator-devlake/plugins/core" "github.com/apache/incubator-devlake/plugins/core/dal" @@ -65,11 +64,9 @@ func ConvertBuildRepos(taskCtx core.SubTaskContext) errors.Error { }, Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { jenkinsBuildRepo := inputRow.(*models.JenkinsBuildRepo) - build := &devops.CiCDPipelineRepo{ - DomainEntity: domainlayer.DomainEntity{ - Id: fmt.Sprintf("%s:%s:%d:%s", "jenkins", "JenkinsTask", jenkinsBuildRepo.ConnectionId, - jenkinsBuildRepo.BuildName), - }, + build := &devops.CiCDPipelineCommit{ + PipelineId: fmt.Sprintf("%s:%s:%d:%s", "jenkins", "JenkinsTask", jenkinsBuildRepo.ConnectionId, + jenkinsBuildRepo.BuildName), CommitSha: jenkinsBuildRepo.CommitSha, Branch: jenkinsBuildRepo.Branch, Repo: jenkinsBuildRepo.RepoUrl,
