This is an automated email from the ASF dual-hosted git repository.
likyh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new f4a058b65 refactor: Change Lead Time doesn't include Jenkins
deployment and other data correctness issues - step 1: repo_url (#4870)
f4a058b65 is described below
commit f4a058b655105aa8f2662b9803e78e848776893a
Author: Klesh Wong <[email protected]>
AuthorDate: Fri Apr 7 16:12:22 2023 +0800
refactor: Change Lead Time doesn't include Jenkins deployment and other
data correctness issues - step 1: repo_url (#4870)
* refactor: rename cicd_pipeline_commits.repo to repo_url
* fix: plugin e2e test
* fix: rename cicd_pipeline_commits.repo to repo_url not working
* fix: direct_urn won't apply framework scripts
* fix: bitbucket cli mode not working
* fix: remove unnecessary required subtasks for bitbucket
* feat: bitbucket filling cicd_pipeine_commits.repo_url
---
.../domainlayer/devops/cicd_pipeline_commmit.go | 2 +-
...30406_rename_cicd_pipeline_commits_repo_url.go} | 32 +++++++-----
backend/core/models/migrationscripts/register.go | 1 +
backend/core/runner/directrun.go | 9 ++--
.../e2e/snapshot_tables/cicd_pipeline_commits.csv | 2 +-
backend/plugins/bitbucket/e2e/pipelines_test.go | 4 +-
backend/plugins/bitbucket/impl/impl.go | 2 +-
.../plugins/bitbucket/tasks/account_convertor.go | 5 +-
.../bitbucket/tasks/issue_comment_collector.go | 2 +-
.../bitbucket/tasks/issue_comment_extractor.go | 5 +-
.../plugins/bitbucket/tasks/pipeline_convertor.go | 12 ++++-
backend/plugins/bitbucket/tasks/pr_collector.go | 2 +-
.../bitbucket/tasks/pr_comment_collector.go | 2 +-
.../bitbucket/tasks/pr_comment_extractor.go | 5 +-
backend/plugins/bitbucket/tasks/pr_convertor.go | 5 +-
backend/plugins/bitbucket/tasks/pr_extractor.go | 5 +-
.../dora/e2e/raw_tables/cicd_pipeline_commits.csv | 2 +-
.../e2e/snapshot_tables/cicd_pipeline_commits.csv | 2 +-
.../e2e/snapshot_tables/cicd_pipeline_commits.csv | 2 +-
backend/plugins/jenkins/e2e/builds_test.go | 5 +-
.../e2e/snapshot_tables/cicd_pipeline_commits.csv | 2 +-
.../jenkins/tasks/build_commit_convertor.go | 5 +-
.../e2e/raw_tables/cicd_pipeline_commits.csv | 2 +-
.../snapshot_tables/_tool_trello_check_items.csv | 57 ++++++++++++++++++++++
backend/plugins/webhook/api/cicd_pipeline.go | 9 ++--
25 files changed, 135 insertions(+), 46 deletions(-)
diff --git a/backend/core/models/domainlayer/devops/cicd_pipeline_commmit.go
b/backend/core/models/domainlayer/devops/cicd_pipeline_commmit.go
index 5c6ec3cb1..9818856f4 100644
--- a/backend/core/models/domainlayer/devops/cicd_pipeline_commmit.go
+++ b/backend/core/models/domainlayer/devops/cicd_pipeline_commmit.go
@@ -27,7 +27,7 @@ type CiCDPipelineCommit struct {
CommitSha string `gorm:"primaryKey;type:varchar(255)"`
Branch string `gorm:"type:varchar(255)"`
RepoId string `gorm:"index;type:varchar(255)"`
- Repo string
+ RepoUrl string
}
func (CiCDPipelineCommit) TableName() string {
diff --git a/backend/core/models/domainlayer/devops/cicd_pipeline_commmit.go
b/backend/core/models/migrationscripts/20230406_rename_cicd_pipeline_commits_repo_url.go
similarity index 52%
copy from backend/core/models/domainlayer/devops/cicd_pipeline_commmit.go
copy to
backend/core/models/migrationscripts/20230406_rename_cicd_pipeline_commits_repo_url.go
index 5c6ec3cb1..4cb45b7f8 100644
--- a/backend/core/models/domainlayer/devops/cicd_pipeline_commmit.go
+++
b/backend/core/models/migrationscripts/20230406_rename_cicd_pipeline_commits_repo_url.go
@@ -15,21 +15,31 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package devops
+package migrationscripts
import (
- "github.com/apache/incubator-devlake/core/models/common"
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
)
-type CiCDPipelineCommit struct {
- common.NoPKModel
- PipelineId string `gorm:"primaryKey;type:varchar(255)"`
- CommitSha string `gorm:"primaryKey;type:varchar(255)"`
- Branch string `gorm:"type:varchar(255)"`
- RepoId string `gorm:"index;type:varchar(255)"`
- Repo string
+var _ plugin.MigrationScript = (*renameCicdPipelineRepoToRepoUrl)(nil)
+
+type renameCicdPipelineRepoToRepoUrl struct{}
+
+func (*renameCicdPipelineRepoToRepoUrl) Up(basicRes context.BasicRes)
errors.Error {
+ db := basicRes.GetDal()
+ err := db.RenameColumn("cicd_pipeline_commits", "repo", "repo_url")
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func (*renameCicdPipelineRepoToRepoUrl) Version() uint64 {
+ return 20230406170701
}
-func (CiCDPipelineCommit) TableName() string {
- return "cicd_pipeline_commits"
+func (*renameCicdPipelineRepoToRepoUrl) Name() string {
+ return "Rename cicd_piopeline_commits repo to repo_url"
}
diff --git a/backend/core/models/migrationscripts/register.go
b/backend/core/models/migrationscripts/register.go
index 159cc9821..9e858bff0 100644
--- a/backend/core/models/migrationscripts/register.go
+++ b/backend/core/models/migrationscripts/register.go
@@ -76,5 +76,6 @@ func All() []plugin.MigrationScript {
new(removeCreatedDateAfterFromCollectorMeta20230223),
new(addHostNamespaceRepoName),
new(renameCollectorTapStateTable),
+ new(renameCicdPipelineRepoToRepoUrl),
}
}
diff --git a/backend/core/runner/directrun.go b/backend/core/runner/directrun.go
index 8ff37f5c5..5c9f49426 100644
--- a/backend/core/runner/directrun.go
+++ b/backend/core/runner/directrun.go
@@ -22,14 +22,16 @@ import (
"encoding/json"
goerror "errors"
"fmt"
- "github.com/apache/incubator-devlake/core/models"
- "github.com/apache/incubator-devlake/core/plugin"
- "github.com/spf13/cobra"
"io"
"os"
"os/signal"
"runtime"
"syscall"
+
+ "github.com/apache/incubator-devlake/core/models"
+ "github.com/apache/incubator-devlake/core/models/migrationscripts"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/spf13/cobra"
)
// RunCmd FIXME ...
@@ -69,6 +71,7 @@ func DirectRun(cmd *cobra.Command, args []string, pluginTask
plugin.PluginTask,
if err != nil {
panic(err)
}
+ migrator.Register(migrationscripts.All(), "Framework")
if migratable, ok := pluginTask.(plugin.PluginMigration); ok {
migrator.Register(migratable.MigrationScripts(), cmd.Use)
}
diff --git
a/backend/plugins/bamboo/e2e/snapshot_tables/cicd_pipeline_commits.csv
b/backend/plugins/bamboo/e2e/snapshot_tables/cicd_pipeline_commits.csv
index 423faec8e..ff994bc63 100644
--- a/backend/plugins/bamboo/e2e/snapshot_tables/cicd_pipeline_commits.csv
+++ b/backend/plugins/bamboo/e2e/snapshot_tables/cicd_pipeline_commits.csv
@@ -1 +1 @@
-pipeline_id,commit_sha,branch,repo_id,repo
+pipeline_id,commit_sha,branch,repo_id,repo_url
diff --git a/backend/plugins/bitbucket/e2e/pipelines_test.go
b/backend/plugins/bitbucket/e2e/pipelines_test.go
index 0be93e409..524d82b08 100644
--- a/backend/plugins/bitbucket/e2e/pipelines_test.go
+++ b/backend/plugins/bitbucket/e2e/pipelines_test.go
@@ -18,9 +18,10 @@ limitations under the License.
package e2e
import (
- "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"testing"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
+
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/plugins/bitbucket/impl"
"github.com/apache/incubator-devlake/plugins/bitbucket/models"
@@ -65,6 +66,7 @@ func TestBitbucketPipelineDataFlow(t *testing.T) {
// verify conversion
dataflowTester.FlushTabler(&devops.CiCDPipelineCommit{})
dataflowTester.FlushTabler(&devops.CICDPipeline{})
+
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_bitbucket_repos.csv",
&models.BitbucketRepo{})
dataflowTester.Subtask(tasks.ConvertPipelineMeta, taskData)
dataflowTester.VerifyTable(
devops.CICDPipeline{},
diff --git a/backend/plugins/bitbucket/impl/impl.go
b/backend/plugins/bitbucket/impl/impl.go
index c1eb862b3..5d79126f3 100644
--- a/backend/plugins/bitbucket/impl/impl.go
+++ b/backend/plugins/bitbucket/impl/impl.go
@@ -253,7 +253,7 @@ func EnrichOptions(taskCtx plugin.TaskContext,
return err
}
logger.Debug(fmt.Sprintf("Current repo: %s",
repo.FullName))
- scope := repo.ConvertApiScope().(models.BitbucketRepo)
+ scope := repo.ConvertApiScope().(*models.BitbucketRepo)
scope.ConnectionId = op.ConnectionId
err = taskCtx.GetDal().CreateIfNotExist(scope)
if err != nil {
diff --git a/backend/plugins/bitbucket/tasks/account_convertor.go
b/backend/plugins/bitbucket/tasks/account_convertor.go
index 716afc5e3..cd076b8c7 100644
--- a/backend/plugins/bitbucket/tasks/account_convertor.go
+++ b/backend/plugins/bitbucket/tasks/account_convertor.go
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -26,7 +28,6 @@ import (
plugin "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
bitbucketModels
"github.com/apache/incubator-devlake/plugins/bitbucket/models"
- "reflect"
)
const RAW_ACCOUNT_TABLE = "bitbucket_api_accounts"
@@ -35,7 +36,7 @@ var ConvertAccountsMeta = plugin.SubTaskMeta{
Name: "convertAccounts",
EntryPoint: ConvertAccounts,
EnabledByDefault: true,
- Required: true,
+ Required: false,
Description: "Convert tool layer table bitbucket_accounts into
domain layer table accounts",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
}
diff --git a/backend/plugins/bitbucket/tasks/issue_comment_collector.go
b/backend/plugins/bitbucket/tasks/issue_comment_collector.go
index 9bc981125..8f85ceace 100644
--- a/backend/plugins/bitbucket/tasks/issue_comment_collector.go
+++ b/backend/plugins/bitbucket/tasks/issue_comment_collector.go
@@ -29,7 +29,7 @@ var CollectApiIssueCommentsMeta = plugin.SubTaskMeta{
Name: "collectApiIssueComments",
EntryPoint: CollectApiIssueComments,
EnabledByDefault: true,
- Required: true,
+ Required: false,
Description: "Collect issue comments data from bitbucket api",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
}
diff --git a/backend/plugins/bitbucket/tasks/issue_comment_extractor.go
b/backend/plugins/bitbucket/tasks/issue_comment_extractor.go
index 54403a597..3adc0c749 100644
--- a/backend/plugins/bitbucket/tasks/issue_comment_extractor.go
+++ b/backend/plugins/bitbucket/tasks/issue_comment_extractor.go
@@ -19,11 +19,12 @@ package tasks
import (
"encoding/json"
+ "time"
+
"github.com/apache/incubator-devlake/core/errors"
plugin "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/bitbucket/models"
- "time"
)
type BitbucketIssueCommentsResponse struct {
@@ -48,7 +49,7 @@ var ExtractApiIssueCommentsMeta = plugin.SubTaskMeta{
Name: "extractApiIssueComments",
EntryPoint: ExtractApiIssueComments,
EnabledByDefault: true,
- Required: true,
+ Required: false,
Description: "Extract raw issue comments data into tool layer
table BitbucketIssueComments",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
}
diff --git a/backend/plugins/bitbucket/tasks/pipeline_convertor.go
b/backend/plugins/bitbucket/tasks/pipeline_convertor.go
index d0f9edd0d..e715ee438 100644
--- a/backend/plugins/bitbucket/tasks/pipeline_convertor.go
+++ b/backend/plugins/bitbucket/tasks/pipeline_convertor.go
@@ -18,6 +18,9 @@ limitations under the License.
package tasks
import (
+ "reflect"
+ "time"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -26,8 +29,6 @@ import (
plugin "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/bitbucket/models"
- "reflect"
- "time"
)
var ConvertPipelineMeta = plugin.SubTaskMeta{
@@ -42,6 +43,12 @@ func ConvertPipelines(taskCtx plugin.SubTaskContext)
errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_PIPELINE_TABLE)
db := taskCtx.GetDal()
+ repo := &models.BitbucketRepo{}
+ err := db.First(repo, dal.Where("connection_id = ? AND bitbucket_id =
?", data.Options.ConnectionId, data.Options.FullName))
+ if err != nil {
+ return err
+ }
+
cursor, err := db.Cursor(dal.From(models.BitbucketPipeline{}))
if err != nil {
return err
@@ -68,6 +75,7 @@ func ConvertPipelines(taskCtx plugin.SubTaskContext)
errors.Error {
Generate(bitbucketPipeline.ConnectionId, bitbucketPipeline.RepoId),
CommitSha: bitbucketPipeline.CommitSha,
Branch: bitbucketPipeline.RefName,
+ RepoUrl: repo.HTMLUrl,
}
domainPipeline := &devops.CICDPipeline{
DomainEntity: domainlayer.DomainEntity{
diff --git a/backend/plugins/bitbucket/tasks/pr_collector.go
b/backend/plugins/bitbucket/tasks/pr_collector.go
index f804eb2af..aab2c6b13 100644
--- a/backend/plugins/bitbucket/tasks/pr_collector.go
+++ b/backend/plugins/bitbucket/tasks/pr_collector.go
@@ -31,7 +31,7 @@ var CollectApiPullRequestsMeta = plugin.SubTaskMeta{
Name: "collectApiPullRequests",
EntryPoint: CollectApiPullRequests,
EnabledByDefault: true,
- Required: true,
+ Required: false,
Description: "Collect PullRequests data from Bitbucket api",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
}
diff --git a/backend/plugins/bitbucket/tasks/pr_comment_collector.go
b/backend/plugins/bitbucket/tasks/pr_comment_collector.go
index f96a33c7b..70bd44433 100644
--- a/backend/plugins/bitbucket/tasks/pr_comment_collector.go
+++ b/backend/plugins/bitbucket/tasks/pr_comment_collector.go
@@ -29,7 +29,7 @@ var CollectApiPrCommentsMeta = plugin.SubTaskMeta{
Name: "collectApiPullRequestsComments",
EntryPoint: CollectApiPullRequestsComments,
EnabledByDefault: true,
- Required: true,
+ Required: false,
Description: "Collect pull requests comments data from bitbucket
api",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
}
diff --git a/backend/plugins/bitbucket/tasks/pr_comment_extractor.go
b/backend/plugins/bitbucket/tasks/pr_comment_extractor.go
index 84e99ad4a..8042191d7 100644
--- a/backend/plugins/bitbucket/tasks/pr_comment_extractor.go
+++ b/backend/plugins/bitbucket/tasks/pr_comment_extractor.go
@@ -19,18 +19,19 @@ package tasks
import (
"encoding/json"
+ "time"
+
"github.com/apache/incubator-devlake/core/errors"
plugin "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/bitbucket/models"
- "time"
)
var ExtractApiPrCommentsMeta = plugin.SubTaskMeta{
Name: "extractApiPullRequestsComments",
EntryPoint: ExtractApiPullRequestsComments,
EnabledByDefault: true,
- Required: true,
+ Required: false,
Description: "Extract raw pull requests comments data into tool
layer table BitbucketPrComments",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
}
diff --git a/backend/plugins/bitbucket/tasks/pr_convertor.go
b/backend/plugins/bitbucket/tasks/pr_convertor.go
index 0d0877708..82d4ca1eb 100644
--- a/backend/plugins/bitbucket/tasks/pr_convertor.go
+++ b/backend/plugins/bitbucket/tasks/pr_convertor.go
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -26,14 +28,13 @@ import (
plugin "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/bitbucket/models"
- "reflect"
)
var ConvertPullRequestsMeta = plugin.SubTaskMeta{
Name: "convertPullRequests",
EntryPoint: ConvertPullRequests,
EnabledByDefault: true,
- Required: true,
+ Required: false,
Description: "ConvertPullRequests data from Bitbucket api",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
}
diff --git a/backend/plugins/bitbucket/tasks/pr_extractor.go
b/backend/plugins/bitbucket/tasks/pr_extractor.go
index 913da92b6..d96600fc2 100644
--- a/backend/plugins/bitbucket/tasks/pr_extractor.go
+++ b/backend/plugins/bitbucket/tasks/pr_extractor.go
@@ -19,18 +19,19 @@ package tasks
import (
"encoding/json"
+ "time"
+
"github.com/apache/incubator-devlake/core/errors"
plugin "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/bitbucket/models"
- "time"
)
var ExtractApiPullRequestsMeta = plugin.SubTaskMeta{
Name: "extractApiPullRequests",
EntryPoint: ExtractApiPullRequests,
EnabledByDefault: true,
- Required: true,
+ Required: false,
Description: "Extract raw PullRequests data into tool layer table
bitbucket_pull_requests",
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
}
diff --git a/backend/plugins/dora/e2e/raw_tables/cicd_pipeline_commits.csv
b/backend/plugins/dora/e2e/raw_tables/cicd_pipeline_commits.csv
index d66671474..3c5029696 100644
--- a/backend/plugins/dora/e2e/raw_tables/cicd_pipeline_commits.csv
+++ b/backend/plugins/dora/e2e/raw_tables/cicd_pipeline_commits.csv
@@ -1,4 +1,4 @@
-pipeline_id,commit_sha,branch,repo_id,repo
+pipeline_id,commit_sha,branch,repo_id,repo_url
pipeline110,commit300,z5z07j8qDO,repo1,
pipeline111,commit301,7j3t5IFWb6,repo1,
pipeline112,commit302,VBSqoF8WIh,repo1,
diff --git
a/backend/plugins/github/e2e/snapshot_tables/cicd_pipeline_commits.csv
b/backend/plugins/github/e2e/snapshot_tables/cicd_pipeline_commits.csv
index e2d80cf1b..ef7d620b5 100644
--- a/backend/plugins/github/e2e/snapshot_tables/cicd_pipeline_commits.csv
+++ b/backend/plugins/github/e2e/snapshot_tables/cicd_pipeline_commits.csv
@@ -1,4 +1,4 @@
-pipeline_id,commit_sha,branch,repo_id,repo
+pipeline_id,commit_sha,branch,repo_id,repo_url
github:GithubRun:1:134018330:2559400712,5dd23ddff8621e6ae36eb24b20d4c4a06dd73dc9,Fix_rm_redundancy_code,github:GithubRepo:1:134018330,
github:GithubRun:1:134018330:2559400713,5dd23ddff8621e6ae36eb24b20d4c4a06dd73dc9,Fix_rm_redundancy_code,github:GithubRepo:1:134018330,
github:GithubRun:1:134018330:2559400714,5dd23ddff8621e6ae36eb24b20d4c4a06dd73dc9,Fix_rm_redundancy_code,github:GithubRepo:1:134018330,
diff --git
a/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipeline_commits.csv
b/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipeline_commits.csv
index a8b1e9faf..d34b65eb3 100644
--- a/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipeline_commits.csv
+++ b/backend/plugins/gitlab/e2e/snapshot_tables/cicd_pipeline_commits.csv
@@ -1,4 +1,4 @@
-pipeline_id,commit_sha,branch,repo_id,repo
+pipeline_id,commit_sha,branch,repo_id,repo_url
gitlab:GitlabPipeline:1:457474837,b1b82852d48b516a18e56c5bab0ebf54b8f4ccfd,renovate/pin-dependencies,gitlab:GitlabProject:1:12345678,
gitlab:GitlabPipeline:1:457474996,739ab912e6e1ed27cecd8a2d00bfd6fa52afd90a,renovate/jest-monorepo,gitlab:GitlabProject:1:12345678,
gitlab:GitlabPipeline:1:457475160,44d127e0ab7dbc4bc259b55929c9d00b62fc3bf4,renovate/lodash-monorepo,gitlab:GitlabProject:1:12345678,
diff --git a/backend/plugins/jenkins/e2e/builds_test.go
b/backend/plugins/jenkins/e2e/builds_test.go
index 63e11bb24..2ffc6ee44 100644
--- a/backend/plugins/jenkins/e2e/builds_test.go
+++ b/backend/plugins/jenkins/e2e/builds_test.go
@@ -18,12 +18,13 @@ limitations under the License.
package e2e
import (
+ "testing"
+
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/plugins/jenkins/impl"
"github.com/apache/incubator-devlake/plugins/jenkins/models"
"github.com/apache/incubator-devlake/plugins/jenkins/tasks"
- "testing"
)
func TestJenkinsBuildsDataFlow(t *testing.T) {
@@ -142,7 +143,7 @@ func TestJenkinsBuildsDataFlow(t *testing.T) {
e2ehelper.ColumnWithRawData(
"pipeline_id",
"repo_id",
- "repo",
+ "repo_url",
"branch",
"commit_sha",
),
diff --git
a/backend/plugins/jenkins/e2e/snapshot_tables/cicd_pipeline_commits.csv
b/backend/plugins/jenkins/e2e/snapshot_tables/cicd_pipeline_commits.csv
index dec0c0ceb..1027c0ab0 100644
--- a/backend/plugins/jenkins/e2e/snapshot_tables/cicd_pipeline_commits.csv
+++ b/backend/plugins/jenkins/e2e/snapshot_tables/cicd_pipeline_commits.csv
@@ -1,4 +1,4 @@
-pipeline_id,commit_sha,repo_id,repo,branch,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+pipeline_id,commit_sha,repo_id,repo_url,branch,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#11,ceeffdfdd06bce232f9adb3a656265bad13a8473,,https://github.com/merico-dev/lake.git,refs/remotes/origin/main,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_builds,95,
jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#21,0f886c74949c3ee7e489188911c7dc0c1d547418,,https://github.com/merico-dev/lake.git,refs/remotes/origin/main,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_builds,94,
jenkins:JenkinsBuild:1:Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake#31,0f886c74949c3ee7e489188911c7dc0c1d547418,,https://github.com/merico-dev/lake.git,refs/remotes/origin/main,"{""ConnectionId"":1,""FullName"":""Test-jenkins-dir/test-jenkins-sub-dir/test-sub-sub-dir/devlake""}",_raw_jenkins_api_builds,93,
diff --git a/backend/plugins/jenkins/tasks/build_commit_convertor.go
b/backend/plugins/jenkins/tasks/build_commit_convertor.go
index 652f7e014..f772e0a31 100644
--- a/backend/plugins/jenkins/tasks/build_commit_convertor.go
+++ b/backend/plugins/jenkins/tasks/build_commit_convertor.go
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
@@ -25,7 +27,6 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/jenkins/models"
- "reflect"
)
var ConvertBuildReposMeta = plugin.SubTaskMeta{
@@ -74,7 +75,7 @@ func ConvertBuildRepos(taskCtx plugin.SubTaskContext)
errors.Error {
PipelineId:
buildIdGen.Generate(jenkinsBuildCommit.ConnectionId,
jenkinsBuildCommit.BuildName),
CommitSha: jenkinsBuildCommit.CommitSha,
Branch: jenkinsBuildCommit.Branch,
- Repo: jenkinsBuildCommit.RepoUrl,
+ RepoUrl: jenkinsBuildCommit.RepoUrl,
}
return []interface{}{
build,
diff --git a/backend/plugins/refdiff/e2e/raw_tables/cicd_pipeline_commits.csv
b/backend/plugins/refdiff/e2e/raw_tables/cicd_pipeline_commits.csv
index b32b2ea3b..1567f87d2 100644
--- a/backend/plugins/refdiff/e2e/raw_tables/cicd_pipeline_commits.csv
+++ b/backend/plugins/refdiff/e2e/raw_tables/cicd_pipeline_commits.csv
@@ -1,4 +1,4 @@
-pipeline_id,commit_sha,branch,repo_id,repo
+pipeline_id,commit_sha,branch,repo_id,repo_url
github:GithubRun:1:1,commit_sha1,main,github:GithubRepo:1:484251804,""
github:GithubRun:1:2,commit_sha2,main,github:GithubRepo:1:484251804,""
github:GithubRun:1:3,commit_sha3,fix-edit-url,github:GithubRepo:1:484251804,""
diff --git
a/backend/plugins/trello/e2e/snapshot_tables/_tool_trello_check_items.csv
b/backend/plugins/trello/e2e/snapshot_tables/_tool_trello_check_items.csv
new file mode 100644
index 000000000..aba99b721
--- /dev/null
+++ b/backend/plugins/trello/e2e/snapshot_tables/_tool_trello_check_items.csv
@@ -0,0 +1,57 @@
+id,name,state,id_checklist,checklist_name,id_board,id_card,pos
+6402f644d23aa9af56b2928a,[Example
task],incomplete,6402f643d23aa9af56b29019,To-Do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b28ffd,33751
+6402f644d23aa9af56b2928b,[Another example
task],incomplete,6402f643d23aa9af56b29019,To-Do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b28ffd,50552
+6402f644d23aa9af56b29290,[Example task],complete,6402f643d23aa9af56b2901a,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b28ffd,17309
+6402f644d23aa9af56b29291,[Another example
task],complete,6402f643d23aa9af56b2901a,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b28ffd,34469
+6402f644d23aa9af56b29296,Filter by date
tweeted,incomplete,6402f643d23aa9af56b2901b,To-do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b28ffe,17265
+6402f644d23aa9af56b29297,Create a form to generate tweet
report,incomplete,6402f643d23aa9af56b2901b,To-do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b28ffe,8632.5
+6402f644d23aa9af56b29298,Implement report
functionality,incomplete,6402f643d23aa9af56b2901b,To-do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b28ffe,12948.75
+6402f644d23aa9af56b29299,Download report as
CSV,incomplete,6402f643d23aa9af56b2901b,To-do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b28ffe,34191
+6402f644d23aa9af56b2929a,Download report as
PDF,incomplete,6402f643d23aa9af56b2901b,To-do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b28ffe,50702
+6402f644d23aa9af56b292a8,Create form to register a new
user,incomplete,6402f643d23aa9af56b2901f,To-do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29000,17126
+6402f644d23aa9af56b292a9,Implement functionality to register a new
user,incomplete,6402f643d23aa9af56b2901f,To-do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29000,33572
+6402f644d23aa9af56b292aa,Implement authentication endpoint for mobile app
developer,incomplete,6402f643d23aa9af56b2901f,To-do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29000,17304
+6402f644d23aa9af56b292ab,Implement endpoint to register new
user,incomplete,6402f643d23aa9af56b2901f,To-do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29000,17215
+6402f644d23aa9af56b292b2,Document endpoint on
postman,incomplete,6402f643d23aa9af56b29020,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29000,33845
+6402f644d23aa9af56b292b6,Upload endpoint returns a 400 error
code,incomplete,6402f643d23aa9af56b29021,To-do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29001,17152
+6402f644d23aa9af56b292ba,Implement endpoint to upload
file,complete,6402f643d23aa9af56b29022,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29001,16661
+6402f644d23aa9af56b292bb,Implement endpoint to validate
file,complete,6402f643d23aa9af56b29022,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29001,34005
+6402f644d23aa9af56b292bc,Implement endpoint to tag files in
folders,complete,6402f643d23aa9af56b29022,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29001,67329
+6402f644d23aa9af56b292bd,Implement endpoint to store file on
cloudinary,complete,6402f643d23aa9af56b29022,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29001,132865
+6402f644d23aa9af56b292be,Create a form to send upload
request,incomplete,6402f643d23aa9af56b29022,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29001,198401
+6402f644d23aa9af56b292c6,Implement functionality to send a new
tweet,incomplete,6402f643d23aa9af56b29023,To-do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29002,33572
+6402f644d23aa9af56b292ca,Document endpoint on
postman,complete,6402f643d23aa9af56b29024,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29002,33845
+6402f644d23aa9af56b292cb,Implement endpoint to send new
tweet,complete,6402f643d23aa9af56b29024,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29002,25574.5
+6402f644d23aa9af56b292cc,Create form to send a new
tweet,incomplete,6402f643d23aa9af56b29024,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29002,99381
+6402f644d23aa9af56b292d2,Create like
button,incomplete,6402f643d23aa9af56b29026,To-do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29003,17126
+6402f644d23aa9af56b292d3,Implement functionality to like
tweet,incomplete,6402f643d23aa9af56b29026,To-do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29003,33572
+6402f644d23aa9af56b292d8,Document endpoint on
postman,incomplete,6402f643d23aa9af56b29025,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29003,33845
+6402f644d23aa9af56b292d9,Implement endpoint to like
tweet,complete,6402f643d23aa9af56b29025,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29003,25574.5
+6402f644d23aa9af56b292de,[Example
task],incomplete,6402f643d23aa9af56b29027,To-Do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29004,33751
+6402f644d23aa9af56b292df,[Another example
task],incomplete,6402f643d23aa9af56b29027,To-Do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29004,50552
+6402f644d23aa9af56b292e4,[Example
task],incomplete,6402f643d23aa9af56b29028,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29004,17309
+6402f644d23aa9af56b292e5,[Another example
task],incomplete,6402f643d23aa9af56b29028,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29004,34469
+6402f644d23aa9af56b292ea,[Example
task],incomplete,6402f643d23aa9af56b2902a,To-Do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29005,33751
+6402f644d23aa9af56b292eb,[Another example
task],incomplete,6402f643d23aa9af56b2902a,To-Do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29005,50552
+6402f644d23aa9af56b292f0,[Example
task],incomplete,6402f643d23aa9af56b29029,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29005,17309
+6402f644d23aa9af56b292f1,[Another example
task],incomplete,6402f643d23aa9af56b29029,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29005,34469
+6402f644d23aa9af56b292f6,[Example
task],incomplete,6402f643d23aa9af56b2902b,To-Do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29006,33751
+6402f644d23aa9af56b292f7,[Another example
task],incomplete,6402f643d23aa9af56b2902b,To-Do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29006,50552
+6402f644d23aa9af56b292fc,[Example
task],incomplete,6402f643d23aa9af56b2902c,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29006,17309
+6402f644d23aa9af56b292fd,[Another example
task],incomplete,6402f643d23aa9af56b2902c,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29006,34469
+6402f644d23aa9af56b29302,[Example
task],incomplete,6402f643d23aa9af56b2902d,To-Do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29007,33751
+6402f644d23aa9af56b29303,[Another example
task],incomplete,6402f643d23aa9af56b2902d,To-Do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29007,50552
+6402f644d23aa9af56b29308,[Example
task],incomplete,6402f643d23aa9af56b2902e,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29007,17309
+6402f644d23aa9af56b29309,[Another example
task],incomplete,6402f643d23aa9af56b2902e,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29007,34469
+6402f644d23aa9af56b2930e,[Example
task],incomplete,6402f643d23aa9af56b2902f,To-Do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29008,33751
+6402f644d23aa9af56b2930f,[Another example
task],incomplete,6402f643d23aa9af56b2902f,To-Do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29008,50552
+6402f644d23aa9af56b29314,[Example
task],incomplete,6402f643d23aa9af56b29030,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29008,17309
+6402f644d23aa9af56b29315,[Another example
task],incomplete,6402f643d23aa9af56b29030,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29008,34469
+6402f644d23aa9af56b2931a,[Example
task],incomplete,6402f643d23aa9af56b29032,To-Do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29009,33751
+6402f644d23aa9af56b2931b,[Another example
task],incomplete,6402f643d23aa9af56b29032,To-Do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29009,50552
+6402f644d23aa9af56b29320,[Example
task],incomplete,6402f643d23aa9af56b29031,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29009,17309
+6402f644d23aa9af56b29321,[Another example
task],incomplete,6402f643d23aa9af56b29031,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b29009,34469
+6402f644d23aa9af56b29326,[Example
task],incomplete,6402f643d23aa9af56b29033,To-Do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b2900a,33751
+6402f644d23aa9af56b29327,[Another example
task],incomplete,6402f643d23aa9af56b29033,To-Do
List,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b2900a,50552
+6402f644d23aa9af56b2932c,[Example
task],incomplete,6402f643d23aa9af56b29034,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b2900a,17309
+6402f644d23aa9af56b2932d,[Another example
task],incomplete,6402f643d23aa9af56b29034,Task
Review,6402f643d23aa9af56b28f4b,6402f643d23aa9af56b2900a,34469
diff --git a/backend/plugins/webhook/api/cicd_pipeline.go
b/backend/plugins/webhook/api/cicd_pipeline.go
index c4708d20b..cba9d6cd2 100644
--- a/backend/plugins/webhook/api/cicd_pipeline.go
+++ b/backend/plugins/webhook/api/cicd_pipeline.go
@@ -20,6 +20,10 @@ package api
import (
"crypto/md5"
"fmt"
+ "net/http"
+ "reflect"
+ "time"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -29,9 +33,6 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/webhook/models"
- "net/http"
- "reflect"
- "time"
"github.com/go-playground/validator/v10"
)
@@ -351,7 +352,7 @@ func PostDeploymentCicdTask(input *plugin.ApiResourceInput)
(*plugin.ApiResource
PipelineId: pipelineId,
CommitSha: request.CommitSha,
Branch: ``,
- Repo: request.RepoUrl,
+ RepoUrl: request.RepoUrl,
}
mayRelatedRepo := code.Repo{}
