This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new b80340bf3 feat(plugins): restranformate without api client (#8034)
b80340bf3 is described below
commit b80340bf326c6f5af5a6093e56612822b7fc2d0c
Author: Lynwee <[email protected]>
AuthorDate: Wed Sep 11 19:36:37 2024 +0800
feat(plugins): restranformate without api client (#8034)
* feat(plugins): restranformate without api client
* fix(zentao): fix test errors
* fix(bamboo): fix e2e test
* fix(jira): fix e2e test
* fix(zentao): fix e2e test
* fix(bamboo): fix e2e test
---
backend/plugins/ae/impl/impl.go | 11 ++-
backend/plugins/azuredevops_go/impl/impl.go | 11 ++-
.../plugins/bamboo/e2e/plan_build_commits_test.go | 1 +
backend/plugins/bamboo/impl/impl.go | 13 +++-
.../plugins/bamboo/tasks/plan_commit_convertor.go | 4 +-
backend/plugins/bamboo/tasks/task_data.go | 1 +
backend/plugins/bitbucket/api/blueprint_v200.go | 8 ++-
backend/plugins/bitbucket/impl/impl.go | 48 ++++++++-----
.../plugins/bitbucket_server/api/blueprint_v200.go | 12 ++--
backend/plugins/bitbucket_server/impl/impl.go | 48 ++++++++-----
backend/plugins/feishu/impl/impl.go | 11 ++-
backend/plugins/gitee/impl/impl.go | 11 ++-
backend/plugins/icla/impl/impl.go | 11 ++-
backend/plugins/jenkins/api/blueprint_v200.go | 12 ++--
backend/plugins/jenkins/impl/impl.go | 39 ++++++-----
backend/plugins/jira/api/blueprint_v200.go | 12 ++--
backend/plugins/jira/e2e/changelog_test.go | 1 +
.../snapshot_tables/_tool_jira_server_infos.csv | 2 +
backend/plugins/jira/impl/impl.go | 59 ++++++++++------
.../20240910_add_table_tool_jira_server_info.go} | 32 ++++-----
.../migrationscripts/archived/jira_server_info.go} | 34 ++++------
.../jira/models/migrationscripts/register.go | 1 +
backend/plugins/jira/models/response_type.go | 7 +-
backend/plugins/jira/tasks/account_collector.go | 7 +-
backend/plugins/jira/tasks/epic_collector.go | 14 ++--
.../jira/tasks/issue_changelog_collector.go | 10 ++-
.../jira/tasks/issue_changelog_extractor.go | 8 ++-
backend/plugins/jira/tasks/issue_collector.go | 7 +-
.../plugins/jira/tasks/issue_comment_collector.go | 7 +-
.../plugins/jira/tasks/issue_comment_extractor.go | 7 +-
backend/plugins/jira/tasks/issue_type_collector.go | 7 +-
backend/plugins/jira/tasks/shared.go | 36 ++++++++++
backend/plugins/jira/tasks/sprint_extractor.go | 9 +--
backend/plugins/jira/tasks/task_data.go | 2 +-
backend/plugins/opsgenie/impl/impl.go | 21 +++---
backend/plugins/pagerduty/impl/impl.go | 21 +++---
backend/plugins/slack/impl/impl.go | 11 ++-
backend/plugins/sonarqube/api/blueprint_v200.go | 12 ++--
backend/plugins/sonarqube/impl/impl.go | 40 ++++++-----
backend/plugins/tapd/impl/impl.go | 13 ++--
backend/plugins/teambition/impl/impl.go | 11 ++-
backend/plugins/trello/impl/impl.go | 11 ++-
backend/plugins/zentao/e2e/account_test.go | 7 +-
backend/plugins/zentao/e2e/bug_commits_test.go | 3 +-
backend/plugins/zentao/e2e/bug_test.go | 1 +
backend/plugins/zentao/e2e/changelog_test.go | 9 +--
backend/plugins/zentao/e2e/execution_test.go | 3 +-
backend/plugins/zentao/e2e/story_commits_test.go | 3 +-
backend/plugins/zentao/e2e/story_test.go | 1 +
backend/plugins/zentao/e2e/task_commits_test.go | 3 +-
backend/plugins/zentao/e2e/task_test.go | 1 +
backend/plugins/zentao/impl/impl.go | 79 ++++++++++++++++------
backend/plugins/zentao/impl/impl_test.go | 63 +++++++++++++++++
.../plugins/zentao/tasks/execution_convertor.go | 9 +--
backend/plugins/zentao/tasks/project_convertor.go | 8 +--
backend/plugins/zentao/tasks/shared.go | 16 -----
backend/plugins/zentao/tasks/shared_test.go | 43 ------------
backend/plugins/zentao/tasks/task_data.go | 1 +
58 files changed, 567 insertions(+), 326 deletions(-)
diff --git a/backend/plugins/ae/impl/impl.go b/backend/plugins/ae/impl/impl.go
index 325146bef..389857e1f 100644
--- a/backend/plugins/ae/impl/impl.go
+++ b/backend/plugins/ae/impl/impl.go
@@ -107,9 +107,14 @@ func (p AE) PrepareTaskData(taskCtx plugin.TaskContext,
options map[string]inter
if err != nil {
return nil, errors.Default.Wrap(err, "error getting connection
for AE plugin")
}
- apiClient, err := tasks.CreateApiClient(taskCtx, connection)
- if err != nil {
- return nil, err
+ var apiClient *helper.ApiAsyncClient
+ syncPolicy := taskCtx.SyncPolicy()
+ if !syncPolicy.SkipCollectors {
+ newApiClient, err := tasks.CreateApiClient(taskCtx, connection)
+ if err != nil {
+ return nil, err
+ }
+ apiClient = newApiClient
}
return &tasks.AeTaskData{
Options: &op,
diff --git a/backend/plugins/azuredevops_go/impl/impl.go
b/backend/plugins/azuredevops_go/impl/impl.go
index f82db88e1..c567e49f5 100644
--- a/backend/plugins/azuredevops_go/impl/impl.go
+++ b/backend/plugins/azuredevops_go/impl/impl.go
@@ -127,9 +127,14 @@ func (p Azuredevops) PrepareTaskData(taskCtx
plugin.TaskContext, options map[str
return nil, errors.Default.Wrap(err, "failed to retrieve an
Azure DevOps connection from the database using the provided connection ID")
}
- apiClient, err := tasks.CreateApiClient(taskCtx, connection)
- if err != nil {
- return nil, errors.Default.Wrap(err, "failed to retrieve an
Azure DevOps connection from the database using the provided connection ID")
+ var apiClient *helper.ApiAsyncClient
+ syncPolicy := taskCtx.SyncPolicy()
+ if !syncPolicy.SkipCollectors {
+ newApiClient, err := tasks.CreateApiClient(taskCtx, connection)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "failed to
retrieve an Azure DevOps connection from the database using the provided
connection ID")
+ }
+ apiClient = newApiClient
}
if op.RepositoryId != "" {
diff --git a/backend/plugins/bamboo/e2e/plan_build_commits_test.go
b/backend/plugins/bamboo/e2e/plan_build_commits_test.go
index edbe7f05c..99294fd8b 100644
--- a/backend/plugins/bamboo/e2e/plan_build_commits_test.go
+++ b/backend/plugins/bamboo/e2e/plan_build_commits_test.go
@@ -43,6 +43,7 @@ func TestBambooPlanBuildCommitsDataFlow(t *testing.T) {
},
},
ApiClient: getFakeAPIClient(),
+ EndPoint: getFakeAPIClient().GetEndpoint(),
}
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_bamboo_plan_build_commits.csv",
&models.BambooPlanBuildVcsRevision{})
diff --git a/backend/plugins/bamboo/impl/impl.go
b/backend/plugins/bamboo/impl/impl.go
index bc722c1b9..2aae7ad5e 100644
--- a/backend/plugins/bamboo/impl/impl.go
+++ b/backend/plugins/bamboo/impl/impl.go
@@ -140,10 +140,16 @@ func (p Bamboo) PrepareTaskData(taskCtx
plugin.TaskContext, options map[string]i
if err != nil {
return nil, errors.Default.Wrap(err, "unable to get Bamboo
connection by the given connection ID")
}
+ endPoint := connection.GetEndpoint()
- apiClient, err := tasks.NewBambooApiClient(taskCtx, connection)
- if err != nil {
- return nil, errors.Default.Wrap(err, "unable to get Bamboo API
client instance")
+ var apiClient *helper.ApiAsyncClient
+ syncPolicy := taskCtx.SyncPolicy()
+ if !syncPolicy.SkipCollectors {
+ newApiClient, err := tasks.NewBambooApiClient(taskCtx,
connection)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "unable to get
Bamboo API client instance")
+ }
+ apiClient = newApiClient
}
if op.PlanKey != "" {
var scope *models.BambooPlan
@@ -189,6 +195,7 @@ func (p Bamboo) PrepareTaskData(taskCtx plugin.TaskContext,
options map[string]i
return &tasks.BambooOptions{
Options: op,
ApiClient: apiClient,
+ EndPoint: endPoint,
RegexEnricher: regexEnricher,
}, nil
}
diff --git a/backend/plugins/bamboo/tasks/plan_commit_convertor.go
b/backend/plugins/bamboo/tasks/plan_commit_convertor.go
index 13104d89c..fcbea337e 100644
--- a/backend/plugins/bamboo/tasks/plan_commit_convertor.go
+++ b/backend/plugins/bamboo/tasks/plan_commit_convertor.go
@@ -66,9 +66,9 @@ func ConvertPlanVcs(taskCtx plugin.SubTaskContext)
errors.Error {
Url: line.Url,
}
domainPlanVcs.RepoId = repoMap[line.RepositoryId]
- fakeRepoUrl, err :=
generateFakeRepoUrl(data.ApiClient.GetEndpoint(), line.RepositoryId)
+ fakeRepoUrl, err := generateFakeRepoUrl(data.EndPoint,
line.RepositoryId)
if err != nil {
- logger.Warn(err, "generate fake repo url,
endpoint: %s, repo id: %d", data.ApiClient.GetEndpoint(), line.RepositoryId)
+ logger.Warn(err, "generate fake repo url,
endpoint: %s, repo id: %d", data.EndPoint, line.RepositoryId)
} else {
domainPlanVcs.RepoUrl = fakeRepoUrl
}
diff --git a/backend/plugins/bamboo/tasks/task_data.go
b/backend/plugins/bamboo/tasks/task_data.go
index c5427ac1a..dc240c55f 100644
--- a/backend/plugins/bamboo/tasks/task_data.go
+++ b/backend/plugins/bamboo/tasks/task_data.go
@@ -26,6 +26,7 @@ import (
type BambooOptions struct {
Options *models.BambooOptions
ApiClient *helper.ApiAsyncClient
+ EndPoint string
RegexEnricher *helper.RegexEnricher
}
diff --git a/backend/plugins/bitbucket/api/blueprint_v200.go
b/backend/plugins/bitbucket/api/blueprint_v200.go
index e9ce9f333..3438a039b 100644
--- a/backend/plugins/bitbucket/api/blueprint_v200.go
+++ b/backend/plugins/bitbucket/api/blueprint_v200.go
@@ -54,9 +54,11 @@ func MakeDataSourcePipelinePlanV200(
// needed for the connection to populate its access tokens
// if AppKey authentication method is selected
- _, err = helper.NewApiClientFromConnection(context.TODO(), basicRes,
connection)
- if err != nil {
- return nil, nil, err
+ if !skipCollectors {
+ _, err = helper.NewApiClientFromConnection(context.TODO(),
basicRes, connection)
+ if err != nil {
+ return nil, nil, err
+ }
}
plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails,
connection)
diff --git a/backend/plugins/bitbucket/impl/impl.go
b/backend/plugins/bitbucket/impl/impl.go
index 8d5123620..9a23ab1b3 100644
--- a/backend/plugins/bitbucket/impl/impl.go
+++ b/backend/plugins/bitbucket/impl/impl.go
@@ -154,13 +154,25 @@ func (p Bitbucket) PrepareTaskData(taskCtx
plugin.TaskContext, options map[strin
return nil, errors.Default.Wrap(err, "unable to get bitbucket
connection by the given connection ID")
}
- apiClient, err := tasks.CreateApiClient(taskCtx, connection)
- if err != nil {
- return nil, errors.Default.Wrap(err, "unable to get bitbucket
API client instance")
+ var apiClient *helper.ApiAsyncClient
+ syncPolicy := taskCtx.SyncPolicy()
+ if !syncPolicy.SkipCollectors {
+ newApiClient, err := tasks.CreateApiClient(taskCtx, connection)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "unable to get
bitbucket API client instance")
+ }
+ apiClient = newApiClient
}
- err = EnrichOptions(taskCtx, op, apiClient.ApiClient)
- if err != nil {
- return nil, err
+ if apiClient == nil {
+ err = EnrichOptions(taskCtx, op, nil)
+ if err != nil {
+ return nil, err
+ }
+ } else {
+ err = EnrichOptions(taskCtx, op, apiClient.ApiClient)
+ if err != nil {
+ return nil, err
+ }
}
regexEnricher := helper.NewRegexEnricher()
@@ -276,17 +288,19 @@ func EnrichOptions(taskCtx plugin.TaskContext,
}
} else {
if taskCtx.GetDal().IsErrorNotFound(err) && op.FullName != "" {
- var repo *models.BitbucketApiRepo
- repo, err = tasks.GetApiRepo(op, apiClient)
- if err != nil {
- return err
- }
- logger.Debug(fmt.Sprintf("Current repo: %s",
repo.FullName))
- scope := repo.ConvertApiScope()
- scope.ConnectionId = op.ConnectionId
- err = taskCtx.GetDal().CreateIfNotExist(scope)
- if err != nil {
- return err
+ if apiClient != nil {
+ var repo *models.BitbucketApiRepo
+ repo, err = tasks.GetApiRepo(op, apiClient)
+ if err != nil {
+ return err
+ }
+ logger.Debug(fmt.Sprintf("Current repo: %s",
repo.FullName))
+ scope := repo.ConvertApiScope()
+ scope.ConnectionId = op.ConnectionId
+ err = taskCtx.GetDal().CreateIfNotExist(scope)
+ if err != nil {
+ return err
+ }
}
} else {
return errors.Default.Wrap(err, fmt.Sprintf("fail to
find repo %s", op.FullName))
diff --git a/backend/plugins/bitbucket_server/api/blueprint_v200.go
b/backend/plugins/bitbucket_server/api/blueprint_v200.go
index 21ce03ff6..7c2f6dc39 100644
--- a/backend/plugins/bitbucket_server/api/blueprint_v200.go
+++ b/backend/plugins/bitbucket_server/api/blueprint_v200.go
@@ -52,11 +52,13 @@ func MakeDataSourcePipelinePlanV200(
return nil, nil, err
}
- // needed for the connection to populate its access tokens
- // if AppKey authentication method is selected
- _, err = helper.NewApiClientFromConnection(context.TODO(), basicRes,
connection)
- if err != nil {
- return nil, nil, err
+ if !skipCollectors {
+ // needed for the connection to populate its access tokens
+ // if AppKey authentication method is selected
+ _, err = helper.NewApiClientFromConnection(context.TODO(),
basicRes, connection)
+ if err != nil {
+ return nil, nil, err
+ }
}
plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails,
connection)
diff --git a/backend/plugins/bitbucket_server/impl/impl.go
b/backend/plugins/bitbucket_server/impl/impl.go
index d59c9a9cb..18b4a4679 100644
--- a/backend/plugins/bitbucket_server/impl/impl.go
+++ b/backend/plugins/bitbucket_server/impl/impl.go
@@ -123,13 +123,25 @@ func (p BitbucketServer) PrepareTaskData(taskCtx
plugin.TaskContext, options map
return nil, errors.Default.Wrap(err, "unable to get bitbucket
server connection by the given connection ID")
}
- apiClient, err := tasks.CreateApiClient(taskCtx, connection)
- if err != nil {
- return nil, errors.Default.Wrap(err, "unable to get bitbucket
server API client instance")
+ var apiClient *helper.ApiAsyncClient
+ syncPolicy := taskCtx.SyncPolicy()
+ if !syncPolicy.SkipCollectors {
+ newApiClient, err := tasks.CreateApiClient(taskCtx, connection)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "unable to get
bitbucket server API client instance")
+ }
+ apiClient = newApiClient
}
- err = EnrichOptions(taskCtx, op, apiClient.ApiClient)
- if err != nil {
- return nil, err
+ if apiClient == nil {
+ err = EnrichOptions(taskCtx, op, nil)
+ if err != nil {
+ return nil, err
+ }
+ } else {
+ err = EnrichOptions(taskCtx, op, apiClient.ApiClient)
+ if err != nil {
+ return nil, err
+ }
}
regexEnricher := helper.NewRegexEnricher()
@@ -239,17 +251,19 @@ func EnrichOptions(taskCtx plugin.TaskContext,
}
} else {
if taskCtx.GetDal().IsErrorNotFound(err) && op.FullName != "" {
- var repo *models.BitbucketServerApiRepo
- repo, err = tasks.GetApiRepo(op, apiClient)
- if err != nil {
- return err
- }
- logger.Debug(fmt.Sprintf("Current repo: %s", repo.Slug))
- scope :=
repo.ConvertApiScope().(*models.BitbucketServerRepo)
- scope.ConnectionId = op.ConnectionId
- err = taskCtx.GetDal().CreateIfNotExist(scope)
- if err != nil {
- return err
+ if apiClient != nil {
+ var repo *models.BitbucketServerApiRepo
+ repo, err = tasks.GetApiRepo(op, apiClient)
+ if err != nil {
+ return err
+ }
+ logger.Debug(fmt.Sprintf("Current repo: %s",
repo.Slug))
+ scope :=
repo.ConvertApiScope().(*models.BitbucketServerRepo)
+ scope.ConnectionId = op.ConnectionId
+ err = taskCtx.GetDal().CreateIfNotExist(scope)
+ if err != nil {
+ return err
+ }
}
} else {
return errors.Default.Wrap(err, fmt.Sprintf("fail to
find repo %s", op.FullName))
diff --git a/backend/plugins/feishu/impl/impl.go
b/backend/plugins/feishu/impl/impl.go
index cfe3e1aff..92e0f1571 100644
--- a/backend/plugins/feishu/impl/impl.go
+++ b/backend/plugins/feishu/impl/impl.go
@@ -109,9 +109,14 @@ func (p Feishu) PrepareTaskData(taskCtx
plugin.TaskContext, options map[string]i
return nil, err
}
- apiClient, err := tasks.NewFeishuApiClient(taskCtx, connection)
- if err != nil {
- return nil, err
+ var apiClient *helper.ApiAsyncClient
+ syncPolicy := taskCtx.SyncPolicy()
+ if !syncPolicy.SkipCollectors {
+ newApiClient, err := tasks.NewFeishuApiClient(taskCtx,
connection)
+ if err != nil {
+ return nil, err
+ }
+ apiClient = newApiClient
}
return &tasks.FeishuTaskData{
Options: &op,
diff --git a/backend/plugins/gitee/impl/impl.go
b/backend/plugins/gitee/impl/impl.go
index 59572fe57..35be389e7 100644
--- a/backend/plugins/gitee/impl/impl.go
+++ b/backend/plugins/gitee/impl/impl.go
@@ -162,10 +162,15 @@ func (p Gitee) PrepareTaskData(taskCtx
plugin.TaskContext, options map[string]in
if err != nil {
return nil, err
}
- apiClient, err := tasks.NewGiteeApiClient(taskCtx, connection)
- if err != nil {
- return nil, err
+ var apiClient *helper.ApiAsyncClient
+ syncPolicy := taskCtx.SyncPolicy()
+ if !syncPolicy.SkipCollectors {
+ newApiClient, err := tasks.NewGiteeApiClient(taskCtx,
connection)
+ if err != nil {
+ return nil, err
+ }
+ apiClient = newApiClient
}
return &tasks.GiteeTaskData{
diff --git a/backend/plugins/icla/impl/impl.go
b/backend/plugins/icla/impl/impl.go
index 83be4492f..1f327f313 100644
--- a/backend/plugins/icla/impl/impl.go
+++ b/backend/plugins/icla/impl/impl.go
@@ -71,9 +71,14 @@ func (p Icla) PrepareTaskData(taskCtx plugin.TaskContext,
options map[string]int
return nil, err
}
- apiClient, err := errors.Convert01(tasks.NewIclaApiClient(taskCtx))
- if err != nil {
- return nil, err
+ var apiClient *helper.ApiAsyncClient
+ syncPolicy := taskCtx.SyncPolicy()
+ if !syncPolicy.SkipCollectors {
+ newApiClient, err :=
errors.Convert01(tasks.NewIclaApiClient(taskCtx))
+ if err != nil {
+ return nil, err
+ }
+ apiClient = newApiClient
}
return &tasks.IclaTaskData{
diff --git a/backend/plugins/jenkins/api/blueprint_v200.go
b/backend/plugins/jenkins/api/blueprint_v200.go
index cce0be37d..6f9d1f622 100644
--- a/backend/plugins/jenkins/api/blueprint_v200.go
+++ b/backend/plugins/jenkins/api/blueprint_v200.go
@@ -49,11 +49,13 @@ func MakeDataSourcePipelinePlanV200(
return nil, nil, err
}
- // needed for the connection to populate its access tokens
- // if AppKey authentication method is selected
- _, err = helper.NewApiClientFromConnection(context.TODO(), basicRes,
connection)
- if err != nil {
- return nil, nil, err
+ if !skipCollectors {
+ // needed for the connection to populate its access tokens
+ // if AppKey authentication method is selected
+ _, err = helper.NewApiClientFromConnection(context.TODO(),
basicRes, connection)
+ if err != nil {
+ return nil, nil, err
+ }
}
plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails,
connection)
diff --git a/backend/plugins/jenkins/impl/impl.go
b/backend/plugins/jenkins/impl/impl.go
index 297d9e4b3..7411f7543 100644
--- a/backend/plugins/jenkins/impl/impl.go
+++ b/backend/plugins/jenkins/impl/impl.go
@@ -126,9 +126,14 @@ func (p Jenkins) PrepareTaskData(taskCtx
plugin.TaskContext, options map[string]
return nil, err
}
- apiClient, err := tasks.CreateApiClient(taskCtx, connection)
- if err != nil {
- return nil, err
+ var apiClient *helper.ApiAsyncClient
+ syncPolicy := taskCtx.SyncPolicy()
+ if !syncPolicy.SkipCollectors {
+ newApiClient, err := tasks.CreateApiClient(taskCtx, connection)
+ if err != nil {
+ return nil, err
+ }
+ apiClient = newApiClient
}
op.ConnectionEndpoint = connection.Endpoint
@@ -260,21 +265,23 @@ func EnrichOptions(taskCtx plugin.TaskContext,
}
}
- err = api.GetJob(apiClient, op.JobPath, op.JobName, op.JobFullName,
100, func(job *models.Job, isPath bool) errors.Error {
- log.Debug(fmt.Sprintf("Current job: %s", job.FullName))
- op.JobPath = job.Path
- op.URL = job.URL
- op.Class = job.Class
- jenkinsJob := job.ToJenkinsJob()
+ if apiClient != nil {
+ err = api.GetJob(apiClient, op.JobPath, op.JobName,
op.JobFullName, 100, func(job *models.Job, isPath bool) errors.Error {
+ log.Debug(fmt.Sprintf("Current job: %s", job.FullName))
+ op.JobPath = job.Path
+ op.URL = job.URL
+ op.Class = job.Class
+ jenkinsJob := job.ToJenkinsJob()
- jenkinsJob.ConnectionId = op.ConnectionId
- jenkinsJob.ScopeConfigId = op.ScopeConfigId
+ jenkinsJob.ConnectionId = op.ConnectionId
+ jenkinsJob.ScopeConfigId = op.ScopeConfigId
- err = taskCtx.GetDal().CreateIfNotExist(jenkinsJob)
- return err
- })
- if err != nil {
- return err
+ err = taskCtx.GetDal().CreateIfNotExist(jenkinsJob)
+ return err
+ })
+ if err != nil {
+ return err
+ }
}
if !strings.HasSuffix(op.JobPath, "/") {
diff --git a/backend/plugins/jira/api/blueprint_v200.go
b/backend/plugins/jira/api/blueprint_v200.go
index f60f5cf13..3d840642c 100644
--- a/backend/plugins/jira/api/blueprint_v200.go
+++ b/backend/plugins/jira/api/blueprint_v200.go
@@ -48,11 +48,13 @@ func MakeDataSourcePipelinePlanV200(
return nil, nil, err
}
- // needed for the connection to populate its access tokens
- // if AppKey authentication method is selected
- _, err = helper.NewApiClientFromConnection(context.TODO(), basicRes,
connection)
- if err != nil {
- return nil, nil, err
+ if !skipCollectors {
+ // needed for the connection to populate its access tokens
+ // if AppKey authentication method is selected
+ _, err = helper.NewApiClientFromConnection(context.TODO(),
basicRes, connection)
+ if err != nil {
+ return nil, nil, err
+ }
}
plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails,
connection)
diff --git a/backend/plugins/jira/e2e/changelog_test.go
b/backend/plugins/jira/e2e/changelog_test.go
index 8f6880f00..d375a93ed 100644
--- a/backend/plugins/jira/e2e/changelog_test.go
+++ b/backend/plugins/jira/e2e/changelog_test.go
@@ -39,6 +39,7 @@ func TestIssueChangelogDataFlow(t *testing.T) {
// import raw data table
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_jira_api_issue_changelogs.csv",
"_raw_jira_api_issue_changelogs")
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_jira_issue_fields.csv",
&models.JiraIssueField{})
+
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_jira_server_infos.csv",
&models.JiraServerInfo{})
dataflowTester.FlushTabler(&models.JiraIssueChangelogs{})
dataflowTester.FlushTabler(&models.JiraIssueChangelogItems{})
dataflowTester.FlushTabler(&models.JiraAccount{})
diff --git
a/backend/plugins/jira/e2e/snapshot_tables/_tool_jira_server_infos.csv
b/backend/plugins/jira/e2e/snapshot_tables/_tool_jira_server_infos.csv
new file mode 100644
index 000000000..6704902e0
--- /dev/null
+++ b/backend/plugins/jira/e2e/snapshot_tables/_tool_jira_server_infos.csv
@@ -0,0 +1,2 @@
+"connection_id","base_url","build_date","build_number","deployment_type","scm_info","server_time","server_title","version","version_numbers"
+1,"http://172.26.129.92:8092","2021-04-19T00:00:00.000+0000",816001,"Server","b8b28db1b682e9a8568ad9c3cfad139bae9ed93f","2024-09-10T12:41:46.085+0000","Jira","8.16.1","[8,
16, 1]"
diff --git a/backend/plugins/jira/impl/impl.go
b/backend/plugins/jira/impl/impl.go
index 8a01b7b11..b7a62da20 100644
--- a/backend/plugins/jira/impl/impl.go
+++ b/backend/plugins/jira/impl/impl.go
@@ -19,8 +19,6 @@ package impl
import (
"fmt"
- "net/http"
-
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
@@ -32,6 +30,7 @@ import (
"github.com/apache/incubator-devlake/plugins/jira/models/migrationscripts"
"github.com/apache/incubator-devlake/plugins/jira/tasks"
"github.com/apache/incubator-devlake/plugins/jira/tasks/apiv2models"
+ "net/http"
)
var _ interface {
@@ -190,9 +189,17 @@ func (p Jira) PrepareTaskData(taskCtx plugin.TaskContext,
options map[string]int
if err != nil {
return nil, errors.Default.Wrap(err, "unable to get Jira
connection")
}
- jiraApiClient, err := tasks.NewJiraApiClient(taskCtx, connection)
- if err != nil {
- return nil, errors.Default.Wrap(err, "failed to create jira api
client")
+
+ var apiClient *helper.ApiAsyncClient
+ syncPolicy := taskCtx.SyncPolicy()
+ if !syncPolicy.SkipCollectors {
+ // Jira plugin cannot disable api client when re transforming,
+ // Because it will fetch
+ jiraApiClient, err := tasks.NewJiraApiClient(taskCtx,
connection)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "failed to create
jira api client")
+ }
+ apiClient = jiraApiClient
}
if op.BoardId != 0 {
@@ -202,16 +209,18 @@ func (p Jira) PrepareTaskData(taskCtx plugin.TaskContext,
options map[string]int
db := taskCtx.GetDal()
err = db.First(&scope, dal.Where("connection_id = ? AND
board_id = ?", op.ConnectionId, op.BoardId))
if err != nil && db.IsErrorNotFound(err) {
- var board *apiv2models.Board
- board, err = api.GetApiJira(&op, jiraApiClient)
- if err != nil {
- return nil, err
- }
- logger.Debug(fmt.Sprintf("Current project: %d",
board.ID))
- scope = board.ToToolLayer(connection.ID)
- err = db.CreateIfNotExist(&scope)
- if err != nil {
- return nil, err
+ if apiClient != nil {
+ var board *apiv2models.Board
+ board, err = api.GetApiJira(&op, apiClient)
+ if err != nil {
+ return nil, err
+ }
+ logger.Debug(fmt.Sprintf("Current project: %d",
board.ID))
+ scope = board.ToToolLayer(connection.ID)
+ err = db.CreateIfNotExist(&scope)
+ if err != nil {
+ return nil, err
+ }
}
}
if err != nil {
@@ -241,14 +250,20 @@ func (p Jira) PrepareTaskData(taskCtx plugin.TaskContext,
options map[string]int
op.PageSize = 100
}
- info, code, err := tasks.GetJiraServerInfo(jiraApiClient)
- if err != nil || code != http.StatusOK || info == nil {
- return nil, errors.HttpStatus(code).Wrap(err, "fail to get Jira
server info")
- }
taskData := &tasks.JiraTaskData{
- Options: &op,
- ApiClient: jiraApiClient,
- JiraServerInfo: *info,
+ Options: &op,
+ ApiClient: apiClient,
+ }
+ if taskData.ApiClient != nil {
+ info, code, err := tasks.GetJiraServerInfo(taskData.ApiClient)
+ if err != nil || code != http.StatusOK || info == nil {
+ return nil, errors.HttpStatus(code).Wrap(err, "fail to
get Jira server info")
+ }
+ taskData.JiraServerInfo = info
+ info.ConnectionID = connection.ID
+ if err := taskCtx.GetDal().CreateOrUpdate(info); err != nil {
+ return nil, errors.Default.Wrap(err, "create or update
jira server info")
+ }
}
return taskData, nil
diff --git a/backend/plugins/bamboo/tasks/task_data.go
b/backend/plugins/jira/models/migrationscripts/20240910_add_table_tool_jira_server_info.go
similarity index 57%
copy from backend/plugins/bamboo/tasks/task_data.go
copy to
backend/plugins/jira/models/migrationscripts/20240910_add_table_tool_jira_server_info.go
index c5427ac1a..b7fe36708 100644
--- a/backend/plugins/bamboo/tasks/task_data.go
+++
b/backend/plugins/jira/models/migrationscripts/20240910_add_table_tool_jira_server_info.go
@@ -15,27 +15,27 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package tasks
+package migrationscripts
import (
+ "github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/apache/incubator-devlake/plugins/bamboo/models"
+ "github.com/apache/incubator-devlake/core/plugin"
+
"github.com/apache/incubator-devlake/plugins/jira/models/migrationscripts/archived"
)
-type BambooOptions struct {
- Options *models.BambooOptions
- ApiClient *helper.ApiAsyncClient
- RegexEnricher *helper.RegexEnricher
+var _ plugin.MigrationScript = (*addJiraServerInfo)(nil)
+
+type addJiraServerInfo struct{}
+
+func (*addJiraServerInfo) Up(basicRes context.BasicRes) errors.Error {
+ return basicRes.GetDal().AutoMigrate(archived.JiraServerInfo{})
+}
+
+func (*addJiraServerInfo) Version() uint64 {
+ return 20240910170000
}
-func DecodeAndValidateTaskOptions(options map[string]interface{})
(*models.BambooOptions, errors.Error) {
- var op models.BambooOptions
- if err := helper.Decode(options, &op, nil); err != nil {
- return nil, err
- }
- if op.ConnectionId == 0 {
- return nil, errors.Default.New("connectionId is invalid")
- }
- return &op, nil
+func (*addJiraServerInfo) Name() string {
+ return "add new table _tool_jira_server_info"
}
diff --git a/backend/plugins/bamboo/tasks/task_data.go
b/backend/plugins/jira/models/migrationscripts/archived/jira_server_info.go
similarity index 53%
copy from backend/plugins/bamboo/tasks/task_data.go
copy to
backend/plugins/jira/models/migrationscripts/archived/jira_server_info.go
index c5427ac1a..78a2d4ad1 100644
--- a/backend/plugins/bamboo/tasks/task_data.go
+++ b/backend/plugins/jira/models/migrationscripts/archived/jira_server_info.go
@@ -15,27 +15,21 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package tasks
+package archived
-import (
- "github.com/apache/incubator-devlake/core/errors"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/apache/incubator-devlake/plugins/bamboo/models"
-)
-
-type BambooOptions struct {
- Options *models.BambooOptions
- ApiClient *helper.ApiAsyncClient
- RegexEnricher *helper.RegexEnricher
+type JiraServerInfo struct {
+ ConnectionID uint64 `json:"connection_id" gorm:"primaryKey"`
+ BaseURL string `json:"baseUrl"`
+ BuildDate string `json:"buildDate"`
+ BuildNumber int `json:"buildNumber"`
+ DeploymentType string `json:"deploymentType"`
+ ScmInfo string `json:"ScmInfo"`
+ ServerTime string `json:"serverTime"`
+ ServerTitle string `json:"serverTitle"`
+ Version string `json:"version"`
+ VersionNumbers []int `json:"versionNumbers"
gorm:"type:json;serializer:json"`
}
-func DecodeAndValidateTaskOptions(options map[string]interface{})
(*models.BambooOptions, errors.Error) {
- var op models.BambooOptions
- if err := helper.Decode(options, &op, nil); err != nil {
- return nil, err
- }
- if op.ConnectionId == 0 {
- return nil, errors.Default.New("connectionId is invalid")
- }
- return &op, nil
+func (JiraServerInfo) TableName() string {
+ return "_tool_jira_server_infos"
}
diff --git a/backend/plugins/jira/models/migrationscripts/register.go
b/backend/plugins/jira/models/migrationscripts/register.go
index 5f32aef7c..8d088b262 100644
--- a/backend/plugins/jira/models/migrationscripts/register.go
+++ b/backend/plugins/jira/models/migrationscripts/register.go
@@ -53,5 +53,6 @@ func All() []plugin.MigrationScript {
new(addIssueFieldTable),
new(changeIssueComponentType),
new(flushJiraIssues),
+ new(addJiraServerInfo),
}
}
diff --git a/backend/plugins/jira/models/response_type.go
b/backend/plugins/jira/models/response_type.go
index 368a4bfda..a84478ba1 100644
--- a/backend/plugins/jira/models/response_type.go
+++ b/backend/plugins/jira/models/response_type.go
@@ -25,6 +25,7 @@ const DeploymentServer DeploymentType = "Server"
const LocaleEnUS Locale = "en_US"
type JiraServerInfo struct {
+ ConnectionID uint64 `json:"connection_id" gorm:"primaryKey"`
// for db store
BaseURL string `json:"baseUrl"`
BuildDate string `json:"buildDate"`
BuildNumber int `json:"buildNumber"`
@@ -33,7 +34,11 @@ type JiraServerInfo struct {
ServerTime string `json:"serverTime"`
ServerTitle string `json:"serverTitle"`
Version string `json:"version"`
- VersionNumbers []int `json:"versionNumbers"`
+ VersionNumbers []int `json:"versionNumbers"
gorm:"type:json;serializer:json"`
+}
+
+func (jiraServerInfo JiraServerInfo) IsDeploymentServer() bool {
+ return jiraServerInfo.DeploymentType == DeploymentServer
}
type JiraErrorInfo struct {
diff --git a/backend/plugins/jira/tasks/account_collector.go
b/backend/plugins/jira/tasks/account_collector.go
index fcf85dd6a..2260a4ac4 100644
--- a/backend/plugins/jira/tasks/account_collector.go
+++ b/backend/plugins/jira/tasks/account_collector.go
@@ -66,7 +66,12 @@ func CollectAccounts(taskCtx plugin.SubTaskContext)
errors.Error {
}
queryKey := "accountId"
urlTemplate := "api/2/user"
- if data.JiraServerInfo.DeploymentType == models.DeploymentServer {
+
+ isServerFlag, err := isServer(data.JiraServerInfo, data.ApiClient,
taskCtx.GetDal(), data.Options.ConnectionId)
+ if err != nil {
+ return err
+ }
+ if isServerFlag {
queryKey = "key"
}
diff --git a/backend/plugins/jira/tasks/epic_collector.go
b/backend/plugins/jira/tasks/epic_collector.go
index 959b0e3c8..b6bd8ed8f 100644
--- a/backend/plugins/jira/tasks/epic_collector.go
+++ b/backend/plugins/jira/tasks/epic_collector.go
@@ -22,13 +22,11 @@ import (
"reflect"
"strings"
+ "encoding/json"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/apache/incubator-devlake/plugins/jira/models"
-
- "encoding/json"
"io"
"net/http"
"net/url"
@@ -51,7 +49,11 @@ func CollectEpics(taskCtx plugin.SubTaskContext)
errors.Error {
data := taskCtx.GetData().(*JiraTaskData)
logger := taskCtx.GetLogger()
batchSize := 100
- if data.JiraServerInfo.DeploymentType == models.DeploymentServer &&
len(data.JiraServerInfo.VersionNumbers) == 3 &&
data.JiraServerInfo.VersionNumbers[0] <= 8 {
+ isServerFlag, err := isServer(data.JiraServerInfo, data.ApiClient,
taskCtx.GetDal(), data.Options.ConnectionId)
+ if err != nil {
+ return err
+ }
+ if isServerFlag && len(data.JiraServerInfo.VersionNumbers) == 3 &&
data.JiraServerInfo.VersionNumbers[0] <= 8 {
batchSize = 1
}
epicIterator, err := GetEpicKeysIterator(db, data, batchSize)
@@ -130,12 +132,12 @@ func GetEpicKeysIterator(db dal.Dal, data *JiraTaskData,
batchSize int) (api.Ite
dal.Join(`
LEFT JOIN _tool_jira_board_issues bi ON (
i.connection_id = bi.connection_id
- AND
+ AND
i.issue_id = bi.issue_id
)`),
dal.Where(`
i.connection_id = ?
- AND
+ AND
bi.board_id = ?
AND
i.epic_key != ''
diff --git a/backend/plugins/jira/tasks/issue_changelog_collector.go
b/backend/plugins/jira/tasks/issue_changelog_collector.go
index 76c1eef38..2c286574c 100644
--- a/backend/plugins/jira/tasks/issue_changelog_collector.go
+++ b/backend/plugins/jira/tasks/issue_changelog_collector.go
@@ -29,7 +29,6 @@ import (
"github.com/apache/incubator-devlake/core/log"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/apache/incubator-devlake/plugins/jira/models"
"github.com/apache/incubator-devlake/plugins/jira/tasks/apiv2models"
)
@@ -47,11 +46,16 @@ var CollectIssueChangelogsMeta = plugin.SubTaskMeta{
func CollectIssueChangelogs(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*JiraTaskData)
- if data.JiraServerInfo.DeploymentType == models.DeploymentServer {
+ db := taskCtx.GetDal()
+
+ isServerFlag, err := isServer(data.JiraServerInfo, data.ApiClient, db,
data.Options.ConnectionId)
+ if err != nil {
+ return err
+ }
+ if isServerFlag {
return nil
}
logger := taskCtx.GetLogger()
- db := taskCtx.GetDal()
apiCollector, err := api.NewStatefulApiCollector(api.RawDataSubTaskArgs{
Ctx: taskCtx,
diff --git a/backend/plugins/jira/tasks/issue_changelog_extractor.go
b/backend/plugins/jira/tasks/issue_changelog_extractor.go
index a49142e3f..0cd9d4f65 100644
--- a/backend/plugins/jira/tasks/issue_changelog_extractor.go
+++ b/backend/plugins/jira/tasks/issue_changelog_extractor.go
@@ -23,7 +23,6 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/apache/incubator-devlake/plugins/jira/models"
"github.com/apache/incubator-devlake/plugins/jira/tasks/apiv2models"
)
@@ -39,7 +38,12 @@ var ExtractIssueChangelogsMeta = plugin.SubTaskMeta{
func ExtractIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error {
data := subtaskCtx.GetData().(*JiraTaskData)
- if data.JiraServerInfo.DeploymentType == models.DeploymentServer {
+
+ isServerFlag, err := isServer(data.JiraServerInfo, data.ApiClient,
subtaskCtx.GetDal(), data.Options.ConnectionId)
+ if err != nil {
+ return err
+ }
+ if isServerFlag {
return nil
}
connectionId := data.Options.ConnectionId
diff --git a/backend/plugins/jira/tasks/issue_collector.go
b/backend/plugins/jira/tasks/issue_collector.go
index 84ac2d727..73f354969 100644
--- a/backend/plugins/jira/tasks/issue_collector.go
+++ b/backend/plugins/jira/tasks/issue_collector.go
@@ -172,7 +172,12 @@ func getTimeZone(taskCtx plugin.SubTaskContext)
(*time.Location, errors.Error) {
var resp *http.Response
var path string
var query url.Values
- if data.JiraServerInfo.DeploymentType == models.DeploymentServer {
+ isServerFlag, err := isServer(data.JiraServerInfo, data.ApiClient,
taskCtx.GetDal(), data.Options.ConnectionId)
+ if err != nil {
+ return nil, err
+ }
+
+ if isServerFlag {
path = "api/2/user"
query = url.Values{"username": []string{conn.Username}}
} else {
diff --git a/backend/plugins/jira/tasks/issue_comment_collector.go
b/backend/plugins/jira/tasks/issue_comment_collector.go
index 5ef2777be..14bff5dd2 100644
--- a/backend/plugins/jira/tasks/issue_comment_collector.go
+++ b/backend/plugins/jira/tasks/issue_comment_collector.go
@@ -29,7 +29,6 @@ import (
"github.com/apache/incubator-devlake/core/log"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/apache/incubator-devlake/plugins/jira/models"
"github.com/apache/incubator-devlake/plugins/jira/tasks/apiv2models"
)
@@ -47,7 +46,11 @@ var CollectIssueCommentsMeta = plugin.SubTaskMeta{
func CollectIssueComments(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*JiraTaskData)
- if data.JiraServerInfo.DeploymentType == models.DeploymentServer {
+ isServerFlag, err := isServer(data.JiraServerInfo, data.ApiClient,
taskCtx.GetDal(), data.Options.ConnectionId)
+ if err != nil {
+ return err
+ }
+ if isServerFlag {
return nil
}
logger := taskCtx.GetLogger()
diff --git a/backend/plugins/jira/tasks/issue_comment_extractor.go
b/backend/plugins/jira/tasks/issue_comment_extractor.go
index e53828853..ada1a42ac 100644
--- a/backend/plugins/jira/tasks/issue_comment_extractor.go
+++ b/backend/plugins/jira/tasks/issue_comment_extractor.go
@@ -22,7 +22,6 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/apache/incubator-devlake/plugins/jira/models"
"github.com/apache/incubator-devlake/plugins/jira/tasks/apiv2models"
)
@@ -38,7 +37,11 @@ var ExtractIssueCommentsMeta = plugin.SubTaskMeta{
func ExtractIssueComments(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*JiraTaskData)
- if data.JiraServerInfo.DeploymentType == models.DeploymentServer {
+ isServerFlag, err := isServer(data.JiraServerInfo, data.ApiClient,
taskCtx.GetDal(), data.Options.ConnectionId)
+ if err != nil {
+ return err
+ }
+ if isServerFlag {
return nil
}
connectionId := data.Options.ConnectionId
diff --git a/backend/plugins/jira/tasks/issue_type_collector.go
b/backend/plugins/jira/tasks/issue_type_collector.go
index c867316b7..5c4432d10 100644
--- a/backend/plugins/jira/tasks/issue_type_collector.go
+++ b/backend/plugins/jira/tasks/issue_type_collector.go
@@ -24,7 +24,6 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/apache/incubator-devlake/plugins/jira/models"
)
const RAW_ISSUE_TYPE_TABLE = "jira_api_issue_types"
@@ -45,7 +44,11 @@ func CollectIssueTypes(taskCtx plugin.SubTaskContext)
errors.Error {
logger.Info("collect issue_types")
urlTemplate := "api/3/issuetype"
- if data.JiraServerInfo.DeploymentType == models.DeploymentServer {
+ isServerFlag, err := isServer(data.JiraServerInfo, data.ApiClient,
taskCtx.GetDal(), data.Options.ConnectionId)
+ if err != nil {
+ return err
+ }
+ if isServerFlag {
urlTemplate = "api/2/issuetype"
}
collector, err := api.NewApiCollector(api.ApiCollectorArgs{
diff --git a/backend/plugins/jira/tasks/shared.go
b/backend/plugins/jira/tasks/shared.go
index cdd118dd1..0fd0e5631 100644
--- a/backend/plugins/jira/tasks/shared.go
+++ b/backend/plugins/jira/tasks/shared.go
@@ -18,9 +18,11 @@ limitations under the License.
package tasks
import (
+ "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/jira/models"
"net/http"
)
@@ -46,3 +48,37 @@ func getStdStatus(statusKey string) string {
return ticket.IN_PROGRESS
}
}
+
+func isServer(jiraServerInfo *models.JiraServerInfo, apiclient
*api.ApiAsyncClient, db dal.Dal, connectionID uint64) (bool, errors.Error) {
+ if jiraServerInfo != nil {
+ return jiraServerInfo.IsDeploymentServer(), nil
+ }
+ // try to fetch jiraServerInfo from remote api
+ if apiclient != nil {
+ info, code, err := GetJiraServerInfo(apiclient)
+ if err != nil || code != http.StatusOK || info == nil {
+ return false, errors.HttpStatus(code).Wrap(err, "fail
to get Jira server info")
+ }
+ return info.IsDeploymentServer(), nil
+ }
+ // fetch from db
+ info, err := getJiraServerInfoFromDB(db, connectionID)
+ if err != nil {
+ return false, err
+ }
+ if info == nil {
+ return false, nil
+ }
+ return info.IsDeploymentServer(), nil
+}
+
+func getJiraServerInfoFromDB(db dal.Dal, connectionID uint64)
(*models.JiraServerInfo, errors.Error) {
+ var info models.JiraServerInfo
+ if err := db.First(&info, dal.Where("connection_id = ?",
connectionID)); err != nil {
+ if db.IsErrorNotFound(err) {
+ return nil, nil
+ }
+ return nil, err
+ }
+ return &info, nil
+}
diff --git a/backend/plugins/jira/tasks/sprint_extractor.go
b/backend/plugins/jira/tasks/sprint_extractor.go
index 21276353d..b1012dcec 100644
--- a/backend/plugins/jira/tasks/sprint_extractor.go
+++ b/backend/plugins/jira/tasks/sprint_extractor.go
@@ -39,9 +39,10 @@ var ExtractSprintsMeta = plugin.SubTaskMeta{
func ExtractSprints(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*JiraTaskData)
- isServer := false
- if data.JiraServerInfo.DeploymentType == models.DeploymentServer {
- isServer = true
+
+ isServerFlag, err := isServer(data.JiraServerInfo, data.ApiClient,
taskCtx.GetDal(), data.Options.ConnectionId)
+ if err != nil {
+ return err
}
extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
@@ -64,7 +65,7 @@ func ExtractSprints(taskCtx plugin.SubTaskContext)
errors.Error {
BoardId: data.Options.BoardId,
SprintId: sprint.ID,
}
- return
[]interface{}{sprint.ToToolLayer(data.Options.ConnectionId, isServer),
&boardSprint}, nil
+ return
[]interface{}{sprint.ToToolLayer(data.Options.ConnectionId, isServerFlag),
&boardSprint}, nil
},
})
diff --git a/backend/plugins/jira/tasks/task_data.go
b/backend/plugins/jira/tasks/task_data.go
index 1b0580396..2f9c12e86 100644
--- a/backend/plugins/jira/tasks/task_data.go
+++ b/backend/plugins/jira/tasks/task_data.go
@@ -36,7 +36,7 @@ type JiraOptions struct {
type JiraTaskData struct {
Options *JiraOptions
ApiClient *api.ApiAsyncClient
- JiraServerInfo models.JiraServerInfo
+ JiraServerInfo *models.JiraServerInfo
}
type JiraApiParams models.JiraApiParams
diff --git a/backend/plugins/opsgenie/impl/impl.go
b/backend/plugins/opsgenie/impl/impl.go
index fc234e7aa..5785d53ff 100644
--- a/backend/plugins/opsgenie/impl/impl.go
+++ b/backend/plugins/opsgenie/impl/impl.go
@@ -123,15 +123,20 @@ func (p Opsgenie) PrepareTaskData(taskCtx
plugin.TaskContext, options map[string
return nil, errors.Default.Wrap(err, "unable to get Opsgenie
connection by the given connection ID")
}
- client, err := helper.NewApiClientFromConnection(taskCtx.GetContext(),
taskCtx, connection)
-
- if err != nil {
- return nil, err
- }
- asyncClient, err := helper.CreateAsyncApiClient(taskCtx, client, nil)
- if err != nil {
- return nil, err
+ var asyncClient *helper.ApiAsyncClient
+ syncPolicy := taskCtx.SyncPolicy()
+ if !syncPolicy.SkipCollectors {
+ client, err :=
helper.NewApiClientFromConnection(taskCtx.GetContext(), taskCtx, connection)
+ if err != nil {
+ return nil, err
+ }
+ newAsyncClient, err := helper.CreateAsyncApiClient(taskCtx,
client, nil)
+ if err != nil {
+ return nil, err
+ }
+ asyncClient = newAsyncClient
}
+
return &tasks.OpsgenieTaskData{
Options: op,
Client: asyncClient,
diff --git a/backend/plugins/pagerduty/impl/impl.go
b/backend/plugins/pagerduty/impl/impl.go
index 8ed6b21b9..b237b30e4 100644
--- a/backend/plugins/pagerduty/impl/impl.go
+++ b/backend/plugins/pagerduty/impl/impl.go
@@ -109,14 +109,19 @@ func (p PagerDuty) PrepareTaskData(taskCtx
plugin.TaskContext, options map[strin
return nil, errors.Default.Wrap(err, "unable to get Pagerduty
connection by the given connection ID")
}
- client, err := helper.NewApiClientFromConnection(taskCtx.GetContext(),
taskCtx, connection)
-
- if err != nil {
- return nil, err
- }
- asyncClient, err := helper.CreateAsyncApiClient(taskCtx, client, nil)
- if err != nil {
- return nil, err
+ var asyncClient *helper.ApiAsyncClient
+ syncPolicy := taskCtx.SyncPolicy()
+ if !syncPolicy.SkipCollectors {
+ client, err :=
helper.NewApiClientFromConnection(taskCtx.GetContext(), taskCtx, connection)
+
+ if err != nil {
+ return nil, err
+ }
+ newAsyncClient, err := helper.CreateAsyncApiClient(taskCtx,
client, nil)
+ if err != nil {
+ return nil, err
+ }
+ asyncClient = newAsyncClient
}
return &tasks.PagerDutyTaskData{
Options: op,
diff --git a/backend/plugins/slack/impl/impl.go
b/backend/plugins/slack/impl/impl.go
index 1b8e40940..8b3de9142 100644
--- a/backend/plugins/slack/impl/impl.go
+++ b/backend/plugins/slack/impl/impl.go
@@ -108,9 +108,14 @@ func (p Slack) PrepareTaskData(taskCtx plugin.TaskContext,
options map[string]in
return nil, err
}
- apiClient, err := tasks.NewSlackApiClient(taskCtx, connection)
- if err != nil {
- return nil, err
+ var apiClient *helper.ApiAsyncClient
+ syncPolicy := taskCtx.SyncPolicy()
+ if !syncPolicy.SkipCollectors {
+ newApiClient, err := tasks.NewSlackApiClient(taskCtx,
connection)
+ if err != nil {
+ return nil, err
+ }
+ apiClient = newApiClient
}
return &tasks.SlackTaskData{
Options: &op,
diff --git a/backend/plugins/sonarqube/api/blueprint_v200.go
b/backend/plugins/sonarqube/api/blueprint_v200.go
index e2995aa65..b129b347b 100644
--- a/backend/plugins/sonarqube/api/blueprint_v200.go
+++ b/backend/plugins/sonarqube/api/blueprint_v200.go
@@ -51,11 +51,13 @@ func MakeDataSourcePipelinePlanV200(
return nil, nil, err
}
- // needed for the connection to populate its access tokens
- // if AppKey authentication method is selected
- _, err = helper.NewApiClientFromConnection(context.TODO(), basicRes,
connection)
- if err != nil {
- return nil, nil, err
+ if !skipCollectors {
+ // needed for the connection to populate its access tokens
+ // if AppKey authentication method is selected
+ _, err = helper.NewApiClientFromConnection(context.TODO(),
basicRes, connection)
+ if err != nil {
+ return nil, nil, err
+ }
}
plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails,
connection)
diff --git a/backend/plugins/sonarqube/impl/impl.go
b/backend/plugins/sonarqube/impl/impl.go
index a364435b1..b6d214032 100644
--- a/backend/plugins/sonarqube/impl/impl.go
+++ b/backend/plugins/sonarqube/impl/impl.go
@@ -126,29 +126,37 @@ func (p Sonarqube) PrepareTaskData(taskCtx
plugin.TaskContext, options map[strin
return nil, errors.Default.Wrap(err, "unable to get Sonarqube
connection by the given connection ID")
}
- apiClient, err := tasks.CreateApiClient(taskCtx, connection)
- if err != nil {
- return nil, errors.Default.Wrap(err, "unable to get Sonarqube
API client instance")
+ var apiClient *helper.ApiAsyncClient
+ syncPolicy := taskCtx.SyncPolicy()
+ if !syncPolicy.SkipCollectors {
+ newApiClient, err := tasks.CreateApiClient(taskCtx, connection)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "unable to get
Sonarqube API client instance")
+ }
+ apiClient = newApiClient
}
taskData := &tasks.SonarqubeTaskData{
Options: op,
ApiClient: apiClient,
TaskStartTime: time.Now(),
}
- // even we have project in _tool_sonaqube_projects, we still need to
collect project to update LastAnalysisDate
- var apiProject *models.SonarqubeApiProject
- apiProject, err = api.GetApiProject(op.ProjectKey, apiClient)
- if err != nil {
- return nil, err
- }
- logger.Debug(fmt.Sprintf("Current project: %s", apiProject.ProjectKey))
- scope := apiProject.ConvertApiScope()
- scope.ConnectionId = op.ConnectionId
- err = taskCtx.GetDal().CreateOrUpdate(&scope)
- if err != nil {
- return nil, err
+ if apiClient != nil {
+ // even we have project in _tool_sonarqube_projects, we still
need to collect project to update LastAnalysisDate
+ var apiProject *models.SonarqubeApiProject
+ apiProject, err = api.GetApiProject(op.ProjectKey, apiClient)
+ if err != nil {
+ return nil, err
+ }
+ logger.Debug(fmt.Sprintf("Current project: %s",
apiProject.ProjectKey))
+ scope := apiProject.ConvertApiScope()
+ scope.ConnectionId = op.ConnectionId
+
+ err = taskCtx.GetDal().CreateOrUpdate(&scope)
+ if err != nil {
+ return nil, err
+ }
+ taskData.LastAnalysisDate =
scope.LastAnalysisDate.ToNullableTime()
}
- taskData.LastAnalysisDate = scope.LastAnalysisDate.ToNullableTime()
return taskData, nil
}
diff --git a/backend/plugins/tapd/impl/impl.go
b/backend/plugins/tapd/impl/impl.go
index 0cd823601..2885a4337 100644
--- a/backend/plugins/tapd/impl/impl.go
+++ b/backend/plugins/tapd/impl/impl.go
@@ -204,9 +204,14 @@ func (p Tapd) PrepareTaskData(taskCtx plugin.TaskContext,
options map[string]int
if connection.RateLimitPerHour == 0 {
connection.RateLimitPerHour = 3600
}
- tapdApiClient, err := tasks.NewTapdApiClient(taskCtx, connection)
- if err != nil {
- return nil, errors.Default.Wrap(err, "failed to create tapd api
client")
+ var apiClient *helper.ApiAsyncClient
+ syncPolicy := taskCtx.SyncPolicy()
+ if !syncPolicy.SkipCollectors {
+ tapdApiClient, err := tasks.NewTapdApiClient(taskCtx,
connection)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "failed to create
tapd api client")
+ }
+ apiClient = tapdApiClient
}
if op.WorkspaceId != 0 {
@@ -238,7 +243,7 @@ func (p Tapd) PrepareTaskData(taskCtx plugin.TaskContext,
options map[string]int
op.CstZone = cstZone
taskData := &tasks.TapdTaskData{
Options: op,
- ApiClient: tapdApiClient,
+ ApiClient: apiClient,
Connection: connection,
}
return taskData, nil
diff --git a/backend/plugins/teambition/impl/impl.go
b/backend/plugins/teambition/impl/impl.go
index d31a58736..662cace0f 100644
--- a/backend/plugins/teambition/impl/impl.go
+++ b/backend/plugins/teambition/impl/impl.go
@@ -142,9 +142,14 @@ func (p Teambition) PrepareTaskData(taskCtx
plugin.TaskContext, options map[stri
return nil, errors.Default.Wrap(err, "unable to get Teambition
connection by the given connection ID")
}
- apiClient, err := tasks.NewTeambitionApiClient(taskCtx, connection)
- if err != nil {
- return nil, errors.Default.Wrap(err, "unable to get Teambition
API client instance")
+ var apiClient *helper.ApiAsyncClient
+ syncPolicy := taskCtx.SyncPolicy()
+ if !syncPolicy.SkipCollectors {
+ newApiClient, err := tasks.NewTeambitionApiClient(taskCtx,
connection)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "unable to get
Teambition API client instance")
+ }
+ apiClient = newApiClient
}
taskData := &tasks.TeambitionTaskData{
Options: op,
diff --git a/backend/plugins/trello/impl/impl.go
b/backend/plugins/trello/impl/impl.go
index 35adaa2c1..b3a0f088d 100644
--- a/backend/plugins/trello/impl/impl.go
+++ b/backend/plugins/trello/impl/impl.go
@@ -116,9 +116,14 @@ func (p Trello) PrepareTaskData(taskCtx
plugin.TaskContext, options map[string]i
if err != nil {
return nil, errors.Default.Wrap(err, "error getting connection
for Trello plugin")
}
- apiClient, err := tasks.CreateApiClient(taskCtx, connection)
- if err != nil {
- return nil, err
+ var apiClient *helper.ApiAsyncClient
+ syncPolicy := taskCtx.SyncPolicy()
+ if !syncPolicy.SkipCollectors {
+ newApiClient, err := tasks.CreateApiClient(taskCtx, connection)
+ if err != nil {
+ return nil, err
+ }
+ apiClient = newApiClient
}
return &tasks.TrelloTaskData{
Options: &op,
diff --git a/backend/plugins/zentao/e2e/account_test.go
b/backend/plugins/zentao/e2e/account_test.go
index a42ad4d6e..7e5b05050 100644
--- a/backend/plugins/zentao/e2e/account_test.go
+++ b/backend/plugins/zentao/e2e/account_test.go
@@ -35,7 +35,7 @@ var basicContext = runner.CreateAppBasicRes()
func getFakeAPIClient() *helper.ApiAsyncClient {
client, _ := helper.NewApiClient(gocontext.Background(),
- "https://zentaomax.demo.qucheng.cc/api.php/v1/";,
+ getFakeHomepage()+"/api.php/v1/",
//"https://zentao.demo.haogs.cn/api.php/v1/";,
nil, time.Second*5, "",
basicContext,
@@ -45,6 +45,10 @@ func getFakeAPIClient() *helper.ApiAsyncClient {
}
}
+func getFakeHomepage() string {
+ return "https://zentaomax.demo.qucheng.cc";
+}
+
func TestZentaoAccountDataFlow(t *testing.T) {
var zentao impl.Zentao
@@ -57,6 +61,7 @@ func TestZentaoAccountDataFlow(t *testing.T) {
},
AccountCache: tasks.NewAccountCache(dataflowTester.Dal, 1),
ApiClient: getFakeAPIClient(),
+ HomePageURL: getFakeHomepage(),
}
// import raw data table
diff --git a/backend/plugins/zentao/e2e/bug_commits_test.go
b/backend/plugins/zentao/e2e/bug_commits_test.go
index 0b5ab4b0e..490062e09 100644
--- a/backend/plugins/zentao/e2e/bug_commits_test.go
+++ b/backend/plugins/zentao/e2e/bug_commits_test.go
@@ -38,7 +38,8 @@ func TestZentaoBugCommitsDataFlow(t *testing.T) {
ConnectionId: 1,
ProjectId: 22,
},
- ApiClient: getFakeAPIClient(),
+ ApiClient: getFakeAPIClient(),
+ HomePageURL: getFakeHomepage(),
}
// import _raw_zentao_api_bug_commits raw data table
diff --git a/backend/plugins/zentao/e2e/bug_test.go
b/backend/plugins/zentao/e2e/bug_test.go
index 16dbc6c11..22c5ea616 100644
--- a/backend/plugins/zentao/e2e/bug_test.go
+++ b/backend/plugins/zentao/e2e/bug_test.go
@@ -49,6 +49,7 @@ func TestZentaoBugDataFlow(t *testing.T) {
Bugs: map[int64]struct{}{},
AccountCache: tasks.NewAccountCache(dataflowTester.Dal, 1),
ApiClient: getFakeAPIClient(),
+ HomePageURL: getFakeHomepage(),
}
// import raw data table
diff --git a/backend/plugins/zentao/e2e/changelog_test.go
b/backend/plugins/zentao/e2e/changelog_test.go
index b0d38ed40..901d1a04b 100644
--- a/backend/plugins/zentao/e2e/changelog_test.go
+++ b/backend/plugins/zentao/e2e/changelog_test.go
@@ -43,10 +43,11 @@ func TestZentaoDbGetDataFlow(t *testing.T) {
ConnectionId: 1,
ProjectId: 0,
},
- Stories: map[int64]struct{}{},
- Tasks: map[int64]struct{}{10: {}, 11: {}, 14: {}},
- Bugs: map[int64]struct{}{1: {}, 2: {}, 3: {}, 4: {}},
- ApiClient: getFakeAPIClient(),
+ Stories: map[int64]struct{}{},
+ Tasks: map[int64]struct{}{10: {}, 11: {}, 14: {}},
+ Bugs: map[int64]struct{}{1: {}, 2: {}, 3: {}, 4: {}},
+ ApiClient: getFakeAPIClient(),
+ HomePageURL: getFakeHomepage(),
}
dataflowTester.ImportCsvIntoTabler("./raw_tables/zt_action.csv",
models.ZentaoRemoteDbAction{})
diff --git a/backend/plugins/zentao/e2e/execution_test.go
b/backend/plugins/zentao/e2e/execution_test.go
index 175e819c7..ad724df22 100644
--- a/backend/plugins/zentao/e2e/execution_test.go
+++ b/backend/plugins/zentao/e2e/execution_test.go
@@ -38,7 +38,8 @@ func TestZentaoExecutionDataFlow(t *testing.T) {
ConnectionId: 1,
ProjectId: 192,
},
- ApiClient: getFakeAPIClient(),
+ ApiClient: getFakeAPIClient(),
+ HomePageURL: getFakeHomepage(),
}
// import raw data table
diff --git a/backend/plugins/zentao/e2e/story_commits_test.go
b/backend/plugins/zentao/e2e/story_commits_test.go
index e3932d426..1878edce6 100644
--- a/backend/plugins/zentao/e2e/story_commits_test.go
+++ b/backend/plugins/zentao/e2e/story_commits_test.go
@@ -38,7 +38,8 @@ func TestZentaoStoryCommitsDataFlow(t *testing.T) {
ConnectionId: 1,
ProjectId: 1,
},
- ApiClient: getFakeAPIClient(),
+ ApiClient: getFakeAPIClient(),
+ HomePageURL: getFakeHomepage(),
}
// import _raw_zentao_api_story_commits raw data table
diff --git a/backend/plugins/zentao/e2e/story_test.go
b/backend/plugins/zentao/e2e/story_test.go
index 754d19155..52b0b8d4c 100644
--- a/backend/plugins/zentao/e2e/story_test.go
+++ b/backend/plugins/zentao/e2e/story_test.go
@@ -49,6 +49,7 @@ func TestZentaoStoryDataFlow(t *testing.T) {
Stories: map[int64]struct{}{},
AccountCache: tasks.NewAccountCache(dataflowTester.Dal, 1),
ApiClient: getFakeAPIClient(),
+ HomePageURL: getFakeHomepage(),
}
// import raw data table
diff --git a/backend/plugins/zentao/e2e/task_commits_test.go
b/backend/plugins/zentao/e2e/task_commits_test.go
index ff85f961b..2f392e1e6 100644
--- a/backend/plugins/zentao/e2e/task_commits_test.go
+++ b/backend/plugins/zentao/e2e/task_commits_test.go
@@ -38,7 +38,8 @@ func TestZentaoTaskCommitsDataFlow(t *testing.T) {
ConnectionId: 1,
ProjectId: 48,
},
- ApiClient: getFakeAPIClient(),
+ ApiClient: getFakeAPIClient(),
+ HomePageURL: getFakeHomepage(),
}
// import _raw_zentao_api_task_commits raw data table
diff --git a/backend/plugins/zentao/e2e/task_test.go
b/backend/plugins/zentao/e2e/task_test.go
index a83ec4484..7f99f85a7 100644
--- a/backend/plugins/zentao/e2e/task_test.go
+++ b/backend/plugins/zentao/e2e/task_test.go
@@ -52,6 +52,7 @@ func TestZentaoTaskDataFlow(t *testing.T) {
Tasks: map[int64]struct{}{},
AccountCache: tasks.NewAccountCache(dataflowTester.Dal, 1),
ApiClient: getFakeAPIClient(),
+ HomePageURL: getFakeHomepage(),
}
// import raw data table
diff --git a/backend/plugins/zentao/impl/impl.go
b/backend/plugins/zentao/impl/impl.go
index 4317fa574..5aa74c966 100644
--- a/backend/plugins/zentao/impl/impl.go
+++ b/backend/plugins/zentao/impl/impl.go
@@ -19,6 +19,8 @@ package impl
import (
"fmt"
+ "net/url"
+ "strings"
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
@@ -182,9 +184,14 @@ func (p Zentao) PrepareTaskData(taskCtx
plugin.TaskContext, options map[string]i
return nil, errors.Default.Wrap(err, "unable to get Zentao
connection by the given connection ID: %v")
}
- apiClient, err := tasks.NewZentaoApiClient(taskCtx, connection)
- if err != nil {
- return nil, errors.Default.Wrap(err, "unable to get Zentao API
client instance: %v")
+ var apiClient *helper.ApiAsyncClient
+ syncPolicy := taskCtx.SyncPolicy()
+ if !syncPolicy.SkipCollectors {
+ newApiClient, err := tasks.NewZentaoApiClient(taskCtx,
connection)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "unable to get
Zentao API client instance: %v")
+ }
+ apiClient = newApiClient
}
if op.ScopeConfig == nil && op.ScopeConfigId != 0 {
@@ -203,36 +210,64 @@ func (p Zentao) PrepareTaskData(taskCtx
plugin.TaskContext, options map[string]i
AccountCache: tasks.NewAccountCache(taskCtx.GetDal(),
op.ConnectionId),
}
- if connection.DbUrl != "" {
- if connection.DbLoggingLevel == "" {
- connection.DbLoggingLevel =
taskCtx.GetConfig("DB_LOGGING_LEVEL")
- }
+ if !syncPolicy.SkipCollectors {
+ if connection.DbUrl != "" {
+ if connection.DbLoggingLevel == "" {
+ connection.DbLoggingLevel =
taskCtx.GetConfig("DB_LOGGING_LEVEL")
+ }
- if connection.DbIdleConns == 0 {
- connection.DbIdleConns =
taskCtx.GetConfigReader().GetInt("DB_IDLE_CONNS")
- }
+ if connection.DbIdleConns == 0 {
+ connection.DbIdleConns =
taskCtx.GetConfigReader().GetInt("DB_IDLE_CONNS")
+ }
- if connection.DbMaxConns == 0 {
- connection.DbMaxConns =
taskCtx.GetConfigReader().GetInt("DB_MAX_CONNS")
- }
+ if connection.DbMaxConns == 0 {
+ connection.DbMaxConns =
taskCtx.GetConfigReader().GetInt("DB_MAX_CONNS")
+ }
- v := viper.New()
- v.Set("DB_URL", connection.DbUrl)
- v.Set("DB_LOGGING_LEVEL", connection.DbLoggingLevel)
- v.Set("DB_IDLE_CONNS", connection.DbIdleConns)
- v.Set("DbMaxConns", connection.DbMaxConns)
+ v := viper.New()
+ v.Set("DB_URL", connection.DbUrl)
+ v.Set("DB_LOGGING_LEVEL", connection.DbLoggingLevel)
+ v.Set("DB_IDLE_CONNS", connection.DbIdleConns)
+ v.Set("DbMaxConns", connection.DbMaxConns)
- rgorm, err := runner.NewGormDb(v, taskCtx.GetLogger())
- if err != nil {
- return nil, errors.Default.Wrap(err,
fmt.Sprintf("failed to connect to the zentao remote databases %s",
connection.DbUrl))
+ rgorm, err := runner.NewGormDb(v, taskCtx.GetLogger())
+ if err != nil {
+ return nil, errors.Default.Wrap(err,
fmt.Sprintf("failed to connect to the zentao remote databases %s",
connection.DbUrl))
+ }
+
+ data.RemoteDb = dalgorm.NewDalgorm(rgorm)
}
+ }
- data.RemoteDb = dalgorm.NewDalgorm(rgorm)
+ endpoint := connection.Endpoint
+ if data.ApiClient != nil {
+ endpoint = data.ApiClient.GetEndpoint()
+ }
+ homepage, err := getZentaoHomePage(endpoint)
+ if err != nil {
+ return data, errors.Convert(err)
}
+ data.HomePageURL = homepage
return data, nil
}
+// getZentaoHomePage receive endpoint like
"http://54.158.1.10:30001/api.php/v1/"; and return zentao's homepage like
"http://54.158.1.10:30001/";
+func getZentaoHomePage(endpoint string) (string, error) {
+ if endpoint == "" {
+ return "", errors.Default.New("empty endpoint")
+ }
+ endpointURL, err := url.Parse(endpoint)
+ if err != nil {
+ return "", err
+ } else {
+ protocol := endpointURL.Scheme
+ host := endpointURL.Host
+ zentaoPath, _, _ := strings.Cut(endpointURL.Path, "/api.php/v1")
+ return fmt.Sprintf("%s://%s%s", protocol, host, zentaoPath), nil
+ }
+}
+
// RootPkgPath information lost when compiled as plugin(.so)
func (p Zentao) RootPkgPath() string {
return "github.com/apache/incubator-devlake/plugins/zentao"
diff --git a/backend/plugins/zentao/impl/impl_test.go
b/backend/plugins/zentao/impl/impl_test.go
new file mode 100644
index 000000000..a2330ad3b
--- /dev/null
+++ b/backend/plugins/zentao/impl/impl_test.go
@@ -0,0 +1,63 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package impl
+
+import "testing"
+
+func Test_getZentaoWebURL(t *testing.T) {
+ type args struct {
+ endpoint string
+ }
+ tests := []struct {
+ name string
+ args args
+ want string
+ wantErr bool
+ }{
+ {
+ name: "without-zentao",
+ args: args{endpoint:
"http://54.158.1.10:30001/api.php/v1/"},
+ want: "http://54.158.1.10:30001";,
+ wantErr: false,
+ },
+ {
+ name: "with-zentao",
+ args: args{endpoint:
"http://54.158.1.10:30001/zentao/api.php/v1/"},
+ want: "http://54.158.1.10:30001/zentao";,
+ wantErr: false,
+ },
+ {
+ name: "with-others",
+ args: args{endpoint:
"http://54.158.1.10:30001/abc/api.php/v1/"},
+ want: "http://54.158.1.10:30001/abc";,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := getZentaoHomePage(tt.args.endpoint)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("getZentaoHomePage() error = %v,
wantErr %v", err, tt.wantErr)
+ return
+ }
+ if got != tt.want {
+ t.Errorf("getZentaoHomePage() got = %v, want
%v", got, tt.want)
+ }
+ })
+ }
+}
diff --git a/backend/plugins/zentao/tasks/execution_convertor.go
b/backend/plugins/zentao/tasks/execution_convertor.go
index 41d80d3bb..8fda2676e 100644
--- a/backend/plugins/zentao/tasks/execution_convertor.go
+++ b/backend/plugins/zentao/tasks/execution_convertor.go
@@ -44,7 +44,6 @@ var ConvertExecutionMeta = plugin.SubTaskMeta{
func ConvertExecutions(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
db := taskCtx.GetDal()
- logger := taskCtx.GetLogger()
executionIdGen := didgen.NewDomainIdGenerator(&models.ZentaoExecution{})
projectIdGen := didgen.NewDomainIdGenerator(&models.ZentaoProject{})
cursor, err := db.Cursor(
@@ -54,12 +53,6 @@ func ConvertExecutions(taskCtx plugin.SubTaskContext)
errors.Error {
if err != nil {
return err
}
-
- homePage, getZentaoHomePageErr :=
getZentaoHomePage(data.ApiClient.GetEndpoint())
- if getZentaoHomePageErr != nil {
- logger.Error(getZentaoHomePageErr, "get zentao homepage")
- return errors.Default.WrapRaw(getZentaoHomePageErr)
- }
defer cursor.Close()
convertor, err := api.NewDataConverter(api.DataConverterArgs{
InputRowType: reflect.TypeOf(models.ZentaoExecution{}),
@@ -91,7 +84,7 @@ func ConvertExecutions(taskCtx plugin.SubTaskContext)
errors.Error {
Id:
executionIdGen.Generate(toolExecution.ConnectionId, toolExecution.Id),
},
Name: toolExecution.Name,
- Url:
fmt.Sprintf("%s/execution-view-%d.html", homePage, toolExecution.Id),
+ Url:
fmt.Sprintf("%s/execution-view-%d.html", data.HomePageURL, toolExecution.Id),
Status: domainStatus,
StartedDate:
toolExecution.RealBegan.ToNullableTime(),
EndedDate:
toolExecution.PlanEnd.ToNullableTime(),
diff --git a/backend/plugins/zentao/tasks/project_convertor.go
b/backend/plugins/zentao/tasks/project_convertor.go
index b09ef9405..d45a00758 100644
--- a/backend/plugins/zentao/tasks/project_convertor.go
+++ b/backend/plugins/zentao/tasks/project_convertor.go
@@ -45,7 +45,6 @@ var ConvertProjectMeta = plugin.SubTaskMeta{
func ConvertProjects(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
db := taskCtx.GetDal()
- logger := taskCtx.GetLogger()
boardIdGen := didgen.NewDomainIdGenerator(&models.ZentaoProject{})
cursor, err := db.Cursor(
dal.From(&models.ZentaoProject{}),
@@ -55,11 +54,6 @@ func ConvertProjects(taskCtx plugin.SubTaskContext)
errors.Error {
return err
}
defer cursor.Close()
- homePage, getZentaoHomePageErr :=
getZentaoHomePage(data.ApiClient.GetEndpoint())
- if getZentaoHomePageErr != nil {
- logger.Error(getZentaoHomePageErr, "get zentao homepage")
- return errors.Default.WrapRaw(getZentaoHomePageErr)
- }
convertor, err := api.NewDataConverter(api.DataConverterArgs{
InputRowType: reflect.TypeOf(models.ZentaoProject{}),
Input: cursor,
@@ -79,7 +73,7 @@ func ConvertProjects(taskCtx plugin.SubTaskContext)
errors.Error {
Description: toolProject.Description,
CreatedDate:
toolProject.OpenedDate.ToNullableTime(),
Type: "scrum",
- Url:
fmt.Sprintf("%s/project-index-%d.html", homePage, data.Options.ProjectId),
+ Url:
fmt.Sprintf("%s/project-index-%d.html", data.HomePageURL,
data.Options.ProjectId),
}
results := make([]interface{}, 0)
results = append(results, domainBoard)
diff --git a/backend/plugins/zentao/tasks/shared.go
b/backend/plugins/zentao/tasks/shared.go
index aa1140573..de4de2307 100644
--- a/backend/plugins/zentao/tasks/shared.go
+++ b/backend/plugins/zentao/tasks/shared.go
@@ -339,19 +339,3 @@ func extractIdFromLogComment(logCommentType string,
comment string) ([]string, e
}
return ret, nil
}
-
-// getZentaoHomePage receive endpoint like
"http://54.158.1.10:30001/api.php/v1/"; and return zentao's homepage like
"http://54.158.1.10:30001/";
-func getZentaoHomePage(endpoint string) (string, error) {
- if endpoint == "" {
- return "", errors.Default.New("empty endpoint")
- }
- endpointURL, err := url.Parse(endpoint)
- if err != nil {
- return "", err
- } else {
- protocol := endpointURL.Scheme
- host := endpointURL.Host
- zentaoPath, _, _ := strings.Cut(endpointURL.Path, "/api.php/v1")
- return fmt.Sprintf("%s://%s%s", protocol, host, zentaoPath), nil
- }
-}
diff --git a/backend/plugins/zentao/tasks/shared_test.go
b/backend/plugins/zentao/tasks/shared_test.go
index 3bec1ca23..3dfc5d8eb 100644
--- a/backend/plugins/zentao/tasks/shared_test.go
+++ b/backend/plugins/zentao/tasks/shared_test.go
@@ -152,46 +152,3 @@ func Test_extractIdFromLogComment(t *testing.T) {
})
}
}
-
-func Test_getZentaoWebURL(t *testing.T) {
- type args struct {
- endpoint string
- }
- tests := []struct {
- name string
- args args
- want string
- wantErr bool
- }{
- {
- name: "without-zentao",
- args: args{endpoint:
"http://54.158.1.10:30001/api.php/v1/"},
- want: "http://54.158.1.10:30001";,
- wantErr: false,
- },
- {
- name: "with-zentao",
- args: args{endpoint:
"http://54.158.1.10:30001/zentao/api.php/v1/"},
- want: "http://54.158.1.10:30001/zentao";,
- wantErr: false,
- },
- {
- name: "with-others",
- args: args{endpoint:
"http://54.158.1.10:30001/abc/api.php/v1/"},
- want: "http://54.158.1.10:30001/abc";,
- wantErr: false,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := getZentaoHomePage(tt.args.endpoint)
- if (err != nil) != tt.wantErr {
- t.Errorf("getZentaoHomePage() error = %v,
wantErr %v", err, tt.wantErr)
- return
- }
- if got != tt.want {
- t.Errorf("getZentaoHomePage() got = %v, want
%v", got, tt.want)
- }
- })
- }
-}
diff --git a/backend/plugins/zentao/tasks/task_data.go
b/backend/plugins/zentao/tasks/task_data.go
index 26bc9734c..a6a097f21 100644
--- a/backend/plugins/zentao/tasks/task_data.go
+++ b/backend/plugins/zentao/tasks/task_data.go
@@ -57,6 +57,7 @@ type ZentaoTaskData struct {
Bugs map[int64]struct{}
AccountCache *AccountCache
ApiClient *helper.ApiAsyncClient
+ HomePageURL string
}
func DecodeAndValidateTaskOptions(options map[string]interface{})
(*ZentaoOptions, error) {
