This is an automated email from the ASF dual-hosted git repository. lynwee pushed a commit to branch dev-1 in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 203ebd600bcc1f0f0ad8305d578a75999fd31a4d Author: d4x1 <[email protected]> AuthorDate: Wed Sep 11 18:32:36 2024 +0800 feat(plugins): restranformate without api client --- backend/plugins/ae/impl/impl.go | 11 ++- backend/plugins/azuredevops_go/impl/impl.go | 11 ++- backend/plugins/bamboo/impl/impl.go | 13 +++- .../plugins/bamboo/tasks/plan_commit_convertor.go | 2 +- backend/plugins/bamboo/tasks/task_data.go | 1 + backend/plugins/bitbucket/api/blueprint_v200.go | 8 ++- backend/plugins/bitbucket/impl/impl.go | 48 ++++++++----- .../plugins/bitbucket_server/api/blueprint_v200.go | 12 ++-- backend/plugins/bitbucket_server/impl/impl.go | 48 ++++++++----- backend/plugins/feishu/impl/impl.go | 11 ++- backend/plugins/gitee/impl/impl.go | 11 ++- backend/plugins/icla/impl/impl.go | 11 ++- backend/plugins/jenkins/api/blueprint_v200.go | 12 ++-- backend/plugins/jenkins/impl/impl.go | 39 ++++++----- backend/plugins/jira/api/blueprint_v200.go | 12 ++-- backend/plugins/jira/impl/impl.go | 59 ++++++++++------ .../20240910_add_table_tool_jira_server_info.go} | 32 ++++----- .../migrationscripts/archived/jira_server_info.go} | 34 ++++------ .../jira/models/migrationscripts/register.go | 1 + backend/plugins/jira/models/response_type.go | 7 +- backend/plugins/jira/tasks/account_collector.go | 7 +- backend/plugins/jira/tasks/epic_collector.go | 14 ++-- .../jira/tasks/issue_changelog_collector.go | 10 ++- .../jira/tasks/issue_changelog_extractor.go | 8 ++- backend/plugins/jira/tasks/issue_collector.go | 7 +- .../plugins/jira/tasks/issue_comment_collector.go | 7 +- .../plugins/jira/tasks/issue_comment_extractor.go | 7 +- backend/plugins/jira/tasks/issue_type_collector.go | 7 +- backend/plugins/jira/tasks/shared.go | 36 ++++++++++ backend/plugins/jira/tasks/sprint_extractor.go | 9 +-- backend/plugins/jira/tasks/task_data.go | 2 +- backend/plugins/opsgenie/impl/impl.go | 21 +++--- backend/plugins/pagerduty/impl/impl.go | 21 +++--- backend/plugins/slack/impl/impl.go | 11 ++- backend/plugins/sonarqube/api/blueprint_v200.go | 12 ++-- backend/plugins/sonarqube/impl/impl.go | 40 ++++++----- backend/plugins/tapd/impl/impl.go | 13 ++-- backend/plugins/teambition/impl/impl.go | 11 ++- backend/plugins/trello/impl/impl.go | 11 ++- backend/plugins/zentao/impl/impl.go | 79 ++++++++++++++++------ .../plugins/zentao/tasks/execution_convertor.go | 9 +-- backend/plugins/zentao/tasks/project_convertor.go | 8 +-- backend/plugins/zentao/tasks/shared.go | 16 ----- backend/plugins/zentao/tasks/task_data.go | 1 + 44 files changed, 477 insertions(+), 273 deletions(-) diff --git a/backend/plugins/ae/impl/impl.go b/backend/plugins/ae/impl/impl.go index 325146bef..389857e1f 100644 --- a/backend/plugins/ae/impl/impl.go +++ b/backend/plugins/ae/impl/impl.go @@ -107,9 +107,14 @@ func (p AE) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]inter if err != nil { return nil, errors.Default.Wrap(err, "error getting connection for AE plugin") } - apiClient, err := tasks.CreateApiClient(taskCtx, connection) - if err != nil { - return nil, err + var apiClient *helper.ApiAsyncClient + syncPolicy := taskCtx.SyncPolicy() + if !syncPolicy.SkipCollectors { + newApiClient, err := tasks.CreateApiClient(taskCtx, connection) + if err != nil { + return nil, err + } + apiClient = newApiClient } return &tasks.AeTaskData{ Options: &op, diff --git a/backend/plugins/azuredevops_go/impl/impl.go b/backend/plugins/azuredevops_go/impl/impl.go index f82db88e1..c567e49f5 100644 --- a/backend/plugins/azuredevops_go/impl/impl.go +++ b/backend/plugins/azuredevops_go/impl/impl.go @@ -127,9 +127,14 @@ func (p Azuredevops) PrepareTaskData(taskCtx plugin.TaskContext, options map[str return nil, errors.Default.Wrap(err, "failed to retrieve an Azure DevOps connection from the database using the provided connection ID") } - apiClient, err := tasks.CreateApiClient(taskCtx, connection) - if err != nil { - return nil, errors.Default.Wrap(err, "failed to retrieve an Azure DevOps connection from the database using the provided connection ID") + var apiClient *helper.ApiAsyncClient + syncPolicy := taskCtx.SyncPolicy() + if !syncPolicy.SkipCollectors { + newApiClient, err := tasks.CreateApiClient(taskCtx, connection) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to retrieve an Azure DevOps connection from the database using the provided connection ID") + } + apiClient = newApiClient } if op.RepositoryId != "" { diff --git a/backend/plugins/bamboo/impl/impl.go b/backend/plugins/bamboo/impl/impl.go index bc722c1b9..2aae7ad5e 100644 --- a/backend/plugins/bamboo/impl/impl.go +++ b/backend/plugins/bamboo/impl/impl.go @@ -140,10 +140,16 @@ func (p Bamboo) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i if err != nil { return nil, errors.Default.Wrap(err, "unable to get Bamboo connection by the given connection ID") } + endPoint := connection.GetEndpoint() - apiClient, err := tasks.NewBambooApiClient(taskCtx, connection) - if err != nil { - return nil, errors.Default.Wrap(err, "unable to get Bamboo API client instance") + var apiClient *helper.ApiAsyncClient + syncPolicy := taskCtx.SyncPolicy() + if !syncPolicy.SkipCollectors { + newApiClient, err := tasks.NewBambooApiClient(taskCtx, connection) + if err != nil { + return nil, errors.Default.Wrap(err, "unable to get Bamboo API client instance") + } + apiClient = newApiClient } if op.PlanKey != "" { var scope *models.BambooPlan @@ -189,6 +195,7 @@ func (p Bamboo) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i return &tasks.BambooOptions{ Options: op, ApiClient: apiClient, + EndPoint: endPoint, RegexEnricher: regexEnricher, }, nil } diff --git a/backend/plugins/bamboo/tasks/plan_commit_convertor.go b/backend/plugins/bamboo/tasks/plan_commit_convertor.go index 13104d89c..5c4c0a4ba 100644 --- a/backend/plugins/bamboo/tasks/plan_commit_convertor.go +++ b/backend/plugins/bamboo/tasks/plan_commit_convertor.go @@ -66,7 +66,7 @@ func ConvertPlanVcs(taskCtx plugin.SubTaskContext) errors.Error { Url: line.Url, } domainPlanVcs.RepoId = repoMap[line.RepositoryId] - fakeRepoUrl, err := generateFakeRepoUrl(data.ApiClient.GetEndpoint(), line.RepositoryId) + fakeRepoUrl, err := generateFakeRepoUrl(data.EndPoint, line.RepositoryId) if err != nil { logger.Warn(err, "generate fake repo url, endpoint: %s, repo id: %d", data.ApiClient.GetEndpoint(), line.RepositoryId) } else { diff --git a/backend/plugins/bamboo/tasks/task_data.go b/backend/plugins/bamboo/tasks/task_data.go index c5427ac1a..dc240c55f 100644 --- a/backend/plugins/bamboo/tasks/task_data.go +++ b/backend/plugins/bamboo/tasks/task_data.go @@ -26,6 +26,7 @@ import ( type BambooOptions struct { Options *models.BambooOptions ApiClient *helper.ApiAsyncClient + EndPoint string RegexEnricher *helper.RegexEnricher } diff --git a/backend/plugins/bitbucket/api/blueprint_v200.go b/backend/plugins/bitbucket/api/blueprint_v200.go index e9ce9f333..3438a039b 100644 --- a/backend/plugins/bitbucket/api/blueprint_v200.go +++ b/backend/plugins/bitbucket/api/blueprint_v200.go @@ -54,9 +54,11 @@ func MakeDataSourcePipelinePlanV200( // needed for the connection to populate its access tokens // if AppKey authentication method is selected - _, err = helper.NewApiClientFromConnection(context.TODO(), basicRes, connection) - if err != nil { - return nil, nil, err + if !skipCollectors { + _, err = helper.NewApiClientFromConnection(context.TODO(), basicRes, connection) + if err != nil { + return nil, nil, err + } } plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails, connection) diff --git a/backend/plugins/bitbucket/impl/impl.go b/backend/plugins/bitbucket/impl/impl.go index 8d5123620..9a23ab1b3 100644 --- a/backend/plugins/bitbucket/impl/impl.go +++ b/backend/plugins/bitbucket/impl/impl.go @@ -154,13 +154,25 @@ func (p Bitbucket) PrepareTaskData(taskCtx plugin.TaskContext, options map[strin return nil, errors.Default.Wrap(err, "unable to get bitbucket connection by the given connection ID") } - apiClient, err := tasks.CreateApiClient(taskCtx, connection) - if err != nil { - return nil, errors.Default.Wrap(err, "unable to get bitbucket API client instance") + var apiClient *helper.ApiAsyncClient + syncPolicy := taskCtx.SyncPolicy() + if !syncPolicy.SkipCollectors { + newApiClient, err := tasks.CreateApiClient(taskCtx, connection) + if err != nil { + return nil, errors.Default.Wrap(err, "unable to get bitbucket API client instance") + } + apiClient = newApiClient } - err = EnrichOptions(taskCtx, op, apiClient.ApiClient) - if err != nil { - return nil, err + if apiClient == nil { + err = EnrichOptions(taskCtx, op, nil) + if err != nil { + return nil, err + } + } else { + err = EnrichOptions(taskCtx, op, apiClient.ApiClient) + if err != nil { + return nil, err + } } regexEnricher := helper.NewRegexEnricher() @@ -276,17 +288,19 @@ func EnrichOptions(taskCtx plugin.TaskContext, } } else { if taskCtx.GetDal().IsErrorNotFound(err) && op.FullName != "" { - var repo *models.BitbucketApiRepo - repo, err = tasks.GetApiRepo(op, apiClient) - if err != nil { - return err - } - logger.Debug(fmt.Sprintf("Current repo: %s", repo.FullName)) - scope := repo.ConvertApiScope() - scope.ConnectionId = op.ConnectionId - err = taskCtx.GetDal().CreateIfNotExist(scope) - if err != nil { - return err + if apiClient != nil { + var repo *models.BitbucketApiRepo + repo, err = tasks.GetApiRepo(op, apiClient) + if err != nil { + return err + } + logger.Debug(fmt.Sprintf("Current repo: %s", repo.FullName)) + scope := repo.ConvertApiScope() + scope.ConnectionId = op.ConnectionId + err = taskCtx.GetDal().CreateIfNotExist(scope) + if err != nil { + return err + } } } else { return errors.Default.Wrap(err, fmt.Sprintf("fail to find repo %s", op.FullName)) diff --git a/backend/plugins/bitbucket_server/api/blueprint_v200.go b/backend/plugins/bitbucket_server/api/blueprint_v200.go index 21ce03ff6..7c2f6dc39 100644 --- a/backend/plugins/bitbucket_server/api/blueprint_v200.go +++ b/backend/plugins/bitbucket_server/api/blueprint_v200.go @@ -52,11 +52,13 @@ func MakeDataSourcePipelinePlanV200( return nil, nil, err } - // needed for the connection to populate its access tokens - // if AppKey authentication method is selected - _, err = helper.NewApiClientFromConnection(context.TODO(), basicRes, connection) - if err != nil { - return nil, nil, err + if !skipCollectors { + // needed for the connection to populate its access tokens + // if AppKey authentication method is selected + _, err = helper.NewApiClientFromConnection(context.TODO(), basicRes, connection) + if err != nil { + return nil, nil, err + } } plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails, connection) diff --git a/backend/plugins/bitbucket_server/impl/impl.go b/backend/plugins/bitbucket_server/impl/impl.go index d59c9a9cb..18b4a4679 100644 --- a/backend/plugins/bitbucket_server/impl/impl.go +++ b/backend/plugins/bitbucket_server/impl/impl.go @@ -123,13 +123,25 @@ func (p BitbucketServer) PrepareTaskData(taskCtx plugin.TaskContext, options map return nil, errors.Default.Wrap(err, "unable to get bitbucket server connection by the given connection ID") } - apiClient, err := tasks.CreateApiClient(taskCtx, connection) - if err != nil { - return nil, errors.Default.Wrap(err, "unable to get bitbucket server API client instance") + var apiClient *helper.ApiAsyncClient + syncPolicy := taskCtx.SyncPolicy() + if !syncPolicy.SkipCollectors { + newApiClient, err := tasks.CreateApiClient(taskCtx, connection) + if err != nil { + return nil, errors.Default.Wrap(err, "unable to get bitbucket server API client instance") + } + apiClient = newApiClient } - err = EnrichOptions(taskCtx, op, apiClient.ApiClient) - if err != nil { - return nil, err + if apiClient == nil { + err = EnrichOptions(taskCtx, op, nil) + if err != nil { + return nil, err + } + } else { + err = EnrichOptions(taskCtx, op, apiClient.ApiClient) + if err != nil { + return nil, err + } } regexEnricher := helper.NewRegexEnricher() @@ -239,17 +251,19 @@ func EnrichOptions(taskCtx plugin.TaskContext, } } else { if taskCtx.GetDal().IsErrorNotFound(err) && op.FullName != "" { - var repo *models.BitbucketServerApiRepo - repo, err = tasks.GetApiRepo(op, apiClient) - if err != nil { - return err - } - logger.Debug(fmt.Sprintf("Current repo: %s", repo.Slug)) - scope := repo.ConvertApiScope().(*models.BitbucketServerRepo) - scope.ConnectionId = op.ConnectionId - err = taskCtx.GetDal().CreateIfNotExist(scope) - if err != nil { - return err + if apiClient != nil { + var repo *models.BitbucketServerApiRepo + repo, err = tasks.GetApiRepo(op, apiClient) + if err != nil { + return err + } + logger.Debug(fmt.Sprintf("Current repo: %s", repo.Slug)) + scope := repo.ConvertApiScope().(*models.BitbucketServerRepo) + scope.ConnectionId = op.ConnectionId + err = taskCtx.GetDal().CreateIfNotExist(scope) + if err != nil { + return err + } } } else { return errors.Default.Wrap(err, fmt.Sprintf("fail to find repo %s", op.FullName)) diff --git a/backend/plugins/feishu/impl/impl.go b/backend/plugins/feishu/impl/impl.go index cfe3e1aff..92e0f1571 100644 --- a/backend/plugins/feishu/impl/impl.go +++ b/backend/plugins/feishu/impl/impl.go @@ -109,9 +109,14 @@ func (p Feishu) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i return nil, err } - apiClient, err := tasks.NewFeishuApiClient(taskCtx, connection) - if err != nil { - return nil, err + var apiClient *helper.ApiAsyncClient + syncPolicy := taskCtx.SyncPolicy() + if !syncPolicy.SkipCollectors { + newApiClient, err := tasks.NewFeishuApiClient(taskCtx, connection) + if err != nil { + return nil, err + } + apiClient = newApiClient } return &tasks.FeishuTaskData{ Options: &op, diff --git a/backend/plugins/gitee/impl/impl.go b/backend/plugins/gitee/impl/impl.go index 59572fe57..35be389e7 100644 --- a/backend/plugins/gitee/impl/impl.go +++ b/backend/plugins/gitee/impl/impl.go @@ -162,10 +162,15 @@ func (p Gitee) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]in if err != nil { return nil, err } - apiClient, err := tasks.NewGiteeApiClient(taskCtx, connection) - if err != nil { - return nil, err + var apiClient *helper.ApiAsyncClient + syncPolicy := taskCtx.SyncPolicy() + if !syncPolicy.SkipCollectors { + newApiClient, err := tasks.NewGiteeApiClient(taskCtx, connection) + if err != nil { + return nil, err + } + apiClient = newApiClient } return &tasks.GiteeTaskData{ diff --git a/backend/plugins/icla/impl/impl.go b/backend/plugins/icla/impl/impl.go index 83be4492f..1f327f313 100644 --- a/backend/plugins/icla/impl/impl.go +++ b/backend/plugins/icla/impl/impl.go @@ -71,9 +71,14 @@ func (p Icla) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int return nil, err } - apiClient, err := errors.Convert01(tasks.NewIclaApiClient(taskCtx)) - if err != nil { - return nil, err + var apiClient *helper.ApiAsyncClient + syncPolicy := taskCtx.SyncPolicy() + if !syncPolicy.SkipCollectors { + newApiClient, err := errors.Convert01(tasks.NewIclaApiClient(taskCtx)) + if err != nil { + return nil, err + } + apiClient = newApiClient } return &tasks.IclaTaskData{ diff --git a/backend/plugins/jenkins/api/blueprint_v200.go b/backend/plugins/jenkins/api/blueprint_v200.go index cce0be37d..6f9d1f622 100644 --- a/backend/plugins/jenkins/api/blueprint_v200.go +++ b/backend/plugins/jenkins/api/blueprint_v200.go @@ -49,11 +49,13 @@ func MakeDataSourcePipelinePlanV200( return nil, nil, err } - // needed for the connection to populate its access tokens - // if AppKey authentication method is selected - _, err = helper.NewApiClientFromConnection(context.TODO(), basicRes, connection) - if err != nil { - return nil, nil, err + if !skipCollectors { + // needed for the connection to populate its access tokens + // if AppKey authentication method is selected + _, err = helper.NewApiClientFromConnection(context.TODO(), basicRes, connection) + if err != nil { + return nil, nil, err + } } plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails, connection) diff --git a/backend/plugins/jenkins/impl/impl.go b/backend/plugins/jenkins/impl/impl.go index 297d9e4b3..7411f7543 100644 --- a/backend/plugins/jenkins/impl/impl.go +++ b/backend/plugins/jenkins/impl/impl.go @@ -126,9 +126,14 @@ func (p Jenkins) PrepareTaskData(taskCtx plugin.TaskContext, options map[string] return nil, err } - apiClient, err := tasks.CreateApiClient(taskCtx, connection) - if err != nil { - return nil, err + var apiClient *helper.ApiAsyncClient + syncPolicy := taskCtx.SyncPolicy() + if !syncPolicy.SkipCollectors { + newApiClient, err := tasks.CreateApiClient(taskCtx, connection) + if err != nil { + return nil, err + } + apiClient = newApiClient } op.ConnectionEndpoint = connection.Endpoint @@ -260,21 +265,23 @@ func EnrichOptions(taskCtx plugin.TaskContext, } } - err = api.GetJob(apiClient, op.JobPath, op.JobName, op.JobFullName, 100, func(job *models.Job, isPath bool) errors.Error { - log.Debug(fmt.Sprintf("Current job: %s", job.FullName)) - op.JobPath = job.Path - op.URL = job.URL - op.Class = job.Class - jenkinsJob := job.ToJenkinsJob() + if apiClient != nil { + err = api.GetJob(apiClient, op.JobPath, op.JobName, op.JobFullName, 100, func(job *models.Job, isPath bool) errors.Error { + log.Debug(fmt.Sprintf("Current job: %s", job.FullName)) + op.JobPath = job.Path + op.URL = job.URL + op.Class = job.Class + jenkinsJob := job.ToJenkinsJob() - jenkinsJob.ConnectionId = op.ConnectionId - jenkinsJob.ScopeConfigId = op.ScopeConfigId + jenkinsJob.ConnectionId = op.ConnectionId + jenkinsJob.ScopeConfigId = op.ScopeConfigId - err = taskCtx.GetDal().CreateIfNotExist(jenkinsJob) - return err - }) - if err != nil { - return err + err = taskCtx.GetDal().CreateIfNotExist(jenkinsJob) + return err + }) + if err != nil { + return err + } } if !strings.HasSuffix(op.JobPath, "/") { diff --git a/backend/plugins/jira/api/blueprint_v200.go b/backend/plugins/jira/api/blueprint_v200.go index f60f5cf13..3d840642c 100644 --- a/backend/plugins/jira/api/blueprint_v200.go +++ b/backend/plugins/jira/api/blueprint_v200.go @@ -48,11 +48,13 @@ func MakeDataSourcePipelinePlanV200( return nil, nil, err } - // needed for the connection to populate its access tokens - // if AppKey authentication method is selected - _, err = helper.NewApiClientFromConnection(context.TODO(), basicRes, connection) - if err != nil { - return nil, nil, err + if !skipCollectors { + // needed for the connection to populate its access tokens + // if AppKey authentication method is selected + _, err = helper.NewApiClientFromConnection(context.TODO(), basicRes, connection) + if err != nil { + return nil, nil, err + } } plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails, connection) diff --git a/backend/plugins/jira/impl/impl.go b/backend/plugins/jira/impl/impl.go index 8a01b7b11..b7a62da20 100644 --- a/backend/plugins/jira/impl/impl.go +++ b/backend/plugins/jira/impl/impl.go @@ -19,8 +19,6 @@ package impl import ( "fmt" - "net/http" - "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" @@ -32,6 +30,7 @@ import ( "github.com/apache/incubator-devlake/plugins/jira/models/migrationscripts" "github.com/apache/incubator-devlake/plugins/jira/tasks" "github.com/apache/incubator-devlake/plugins/jira/tasks/apiv2models" + "net/http" ) var _ interface { @@ -190,9 +189,17 @@ func (p Jira) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int if err != nil { return nil, errors.Default.Wrap(err, "unable to get Jira connection") } - jiraApiClient, err := tasks.NewJiraApiClient(taskCtx, connection) - if err != nil { - return nil, errors.Default.Wrap(err, "failed to create jira api client") + + var apiClient *helper.ApiAsyncClient + syncPolicy := taskCtx.SyncPolicy() + if !syncPolicy.SkipCollectors { + // Jira plugin cannot disable api client when re transforming, + // Because it will fetch + jiraApiClient, err := tasks.NewJiraApiClient(taskCtx, connection) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to create jira api client") + } + apiClient = jiraApiClient } if op.BoardId != 0 { @@ -202,16 +209,18 @@ func (p Jira) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int db := taskCtx.GetDal() err = db.First(&scope, dal.Where("connection_id = ? AND board_id = ?", op.ConnectionId, op.BoardId)) if err != nil && db.IsErrorNotFound(err) { - var board *apiv2models.Board - board, err = api.GetApiJira(&op, jiraApiClient) - if err != nil { - return nil, err - } - logger.Debug(fmt.Sprintf("Current project: %d", board.ID)) - scope = board.ToToolLayer(connection.ID) - err = db.CreateIfNotExist(&scope) - if err != nil { - return nil, err + if apiClient != nil { + var board *apiv2models.Board + board, err = api.GetApiJira(&op, apiClient) + if err != nil { + return nil, err + } + logger.Debug(fmt.Sprintf("Current project: %d", board.ID)) + scope = board.ToToolLayer(connection.ID) + err = db.CreateIfNotExist(&scope) + if err != nil { + return nil, err + } } } if err != nil { @@ -241,14 +250,20 @@ func (p Jira) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int op.PageSize = 100 } - info, code, err := tasks.GetJiraServerInfo(jiraApiClient) - if err != nil || code != http.StatusOK || info == nil { - return nil, errors.HttpStatus(code).Wrap(err, "fail to get Jira server info") - } taskData := &tasks.JiraTaskData{ - Options: &op, - ApiClient: jiraApiClient, - JiraServerInfo: *info, + Options: &op, + ApiClient: apiClient, + } + if taskData.ApiClient != nil { + info, code, err := tasks.GetJiraServerInfo(taskData.ApiClient) + if err != nil || code != http.StatusOK || info == nil { + return nil, errors.HttpStatus(code).Wrap(err, "fail to get Jira server info") + } + taskData.JiraServerInfo = info + info.ConnectionID = connection.ID + if err := taskCtx.GetDal().CreateOrUpdate(info); err != nil { + return nil, errors.Default.Wrap(err, "create or update jira server info") + } } return taskData, nil diff --git a/backend/plugins/bamboo/tasks/task_data.go b/backend/plugins/jira/models/migrationscripts/20240910_add_table_tool_jira_server_info.go similarity index 57% copy from backend/plugins/bamboo/tasks/task_data.go copy to backend/plugins/jira/models/migrationscripts/20240910_add_table_tool_jira_server_info.go index c5427ac1a..b7fe36708 100644 --- a/backend/plugins/bamboo/tasks/task_data.go +++ b/backend/plugins/jira/models/migrationscripts/20240910_add_table_tool_jira_server_info.go @@ -15,27 +15,27 @@ See the License for the specific language governing permissions and limitations under the License. */ -package tasks +package migrationscripts import ( + "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/errors" - helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "github.com/apache/incubator-devlake/plugins/bamboo/models" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/jira/models/migrationscripts/archived" ) -type BambooOptions struct { - Options *models.BambooOptions - ApiClient *helper.ApiAsyncClient - RegexEnricher *helper.RegexEnricher +var _ plugin.MigrationScript = (*addJiraServerInfo)(nil) + +type addJiraServerInfo struct{} + +func (*addJiraServerInfo) Up(basicRes context.BasicRes) errors.Error { + return basicRes.GetDal().AutoMigrate(archived.JiraServerInfo{}) +} + +func (*addJiraServerInfo) Version() uint64 { + return 20240910170000 } -func DecodeAndValidateTaskOptions(options map[string]interface{}) (*models.BambooOptions, errors.Error) { - var op models.BambooOptions - if err := helper.Decode(options, &op, nil); err != nil { - return nil, err - } - if op.ConnectionId == 0 { - return nil, errors.Default.New("connectionId is invalid") - } - return &op, nil +func (*addJiraServerInfo) Name() string { + return "add new table _tool_jira_server_info" } diff --git a/backend/plugins/bamboo/tasks/task_data.go b/backend/plugins/jira/models/migrationscripts/archived/jira_server_info.go similarity index 53% copy from backend/plugins/bamboo/tasks/task_data.go copy to backend/plugins/jira/models/migrationscripts/archived/jira_server_info.go index c5427ac1a..78a2d4ad1 100644 --- a/backend/plugins/bamboo/tasks/task_data.go +++ b/backend/plugins/jira/models/migrationscripts/archived/jira_server_info.go @@ -15,27 +15,21 @@ See the License for the specific language governing permissions and limitations under the License. */ -package tasks +package archived -import ( - "github.com/apache/incubator-devlake/core/errors" - helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "github.com/apache/incubator-devlake/plugins/bamboo/models" -) - -type BambooOptions struct { - Options *models.BambooOptions - ApiClient *helper.ApiAsyncClient - RegexEnricher *helper.RegexEnricher +type JiraServerInfo struct { + ConnectionID uint64 `json:"connection_id" gorm:"primaryKey"` + BaseURL string `json:"baseUrl"` + BuildDate string `json:"buildDate"` + BuildNumber int `json:"buildNumber"` + DeploymentType string `json:"deploymentType"` + ScmInfo string `json:"ScmInfo"` + ServerTime string `json:"serverTime"` + ServerTitle string `json:"serverTitle"` + Version string `json:"version"` + VersionNumbers []int `json:"versionNumbers" gorm:"type:json;serializer:json"` } -func DecodeAndValidateTaskOptions(options map[string]interface{}) (*models.BambooOptions, errors.Error) { - var op models.BambooOptions - if err := helper.Decode(options, &op, nil); err != nil { - return nil, err - } - if op.ConnectionId == 0 { - return nil, errors.Default.New("connectionId is invalid") - } - return &op, nil +func (JiraServerInfo) TableName() string { + return "_tool_jira_server_infos" } diff --git a/backend/plugins/jira/models/migrationscripts/register.go b/backend/plugins/jira/models/migrationscripts/register.go index 5f32aef7c..8d088b262 100644 --- a/backend/plugins/jira/models/migrationscripts/register.go +++ b/backend/plugins/jira/models/migrationscripts/register.go @@ -53,5 +53,6 @@ func All() []plugin.MigrationScript { new(addIssueFieldTable), new(changeIssueComponentType), new(flushJiraIssues), + new(addJiraServerInfo), } } diff --git a/backend/plugins/jira/models/response_type.go b/backend/plugins/jira/models/response_type.go index 368a4bfda..a84478ba1 100644 --- a/backend/plugins/jira/models/response_type.go +++ b/backend/plugins/jira/models/response_type.go @@ -25,6 +25,7 @@ const DeploymentServer DeploymentType = "Server" const LocaleEnUS Locale = "en_US" type JiraServerInfo struct { + ConnectionID uint64 `json:"connection_id" gorm:"primaryKey"` // for db store BaseURL string `json:"baseUrl"` BuildDate string `json:"buildDate"` BuildNumber int `json:"buildNumber"` @@ -33,7 +34,11 @@ type JiraServerInfo struct { ServerTime string `json:"serverTime"` ServerTitle string `json:"serverTitle"` Version string `json:"version"` - VersionNumbers []int `json:"versionNumbers"` + VersionNumbers []int `json:"versionNumbers" gorm:"type:json;serializer:json"` +} + +func (jiraServerInfo JiraServerInfo) IsDeploymentServer() bool { + return jiraServerInfo.DeploymentType == DeploymentServer } type JiraErrorInfo struct { diff --git a/backend/plugins/jira/tasks/account_collector.go b/backend/plugins/jira/tasks/account_collector.go index fcf85dd6a..2260a4ac4 100644 --- a/backend/plugins/jira/tasks/account_collector.go +++ b/backend/plugins/jira/tasks/account_collector.go @@ -66,7 +66,12 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { } queryKey := "accountId" urlTemplate := "api/2/user" - if data.JiraServerInfo.DeploymentType == models.DeploymentServer { + + isServerFlag, err := isServer(data.JiraServerInfo, data.ApiClient, taskCtx.GetDal(), data.Options.ConnectionId) + if err != nil { + return err + } + if isServerFlag { queryKey = "key" } diff --git a/backend/plugins/jira/tasks/epic_collector.go b/backend/plugins/jira/tasks/epic_collector.go index 959b0e3c8..b6bd8ed8f 100644 --- a/backend/plugins/jira/tasks/epic_collector.go +++ b/backend/plugins/jira/tasks/epic_collector.go @@ -22,13 +22,11 @@ import ( "reflect" "strings" + "encoding/json" "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "github.com/apache/incubator-devlake/plugins/jira/models" - - "encoding/json" "io" "net/http" "net/url" @@ -51,7 +49,11 @@ func CollectEpics(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*JiraTaskData) logger := taskCtx.GetLogger() batchSize := 100 - if data.JiraServerInfo.DeploymentType == models.DeploymentServer && len(data.JiraServerInfo.VersionNumbers) == 3 && data.JiraServerInfo.VersionNumbers[0] <= 8 { + isServerFlag, err := isServer(data.JiraServerInfo, data.ApiClient, taskCtx.GetDal(), data.Options.ConnectionId) + if err != nil { + return err + } + if isServerFlag && len(data.JiraServerInfo.VersionNumbers) == 3 && data.JiraServerInfo.VersionNumbers[0] <= 8 { batchSize = 1 } epicIterator, err := GetEpicKeysIterator(db, data, batchSize) @@ -130,12 +132,12 @@ func GetEpicKeysIterator(db dal.Dal, data *JiraTaskData, batchSize int) (api.Ite dal.Join(` LEFT JOIN _tool_jira_board_issues bi ON ( i.connection_id = bi.connection_id - AND + AND i.issue_id = bi.issue_id )`), dal.Where(` i.connection_id = ? - AND + AND bi.board_id = ? AND i.epic_key != '' diff --git a/backend/plugins/jira/tasks/issue_changelog_collector.go b/backend/plugins/jira/tasks/issue_changelog_collector.go index 76c1eef38..2c286574c 100644 --- a/backend/plugins/jira/tasks/issue_changelog_collector.go +++ b/backend/plugins/jira/tasks/issue_changelog_collector.go @@ -29,7 +29,6 @@ import ( "github.com/apache/incubator-devlake/core/log" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "github.com/apache/incubator-devlake/plugins/jira/models" "github.com/apache/incubator-devlake/plugins/jira/tasks/apiv2models" ) @@ -47,11 +46,16 @@ var CollectIssueChangelogsMeta = plugin.SubTaskMeta{ func CollectIssueChangelogs(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*JiraTaskData) - if data.JiraServerInfo.DeploymentType == models.DeploymentServer { + db := taskCtx.GetDal() + + isServerFlag, err := isServer(data.JiraServerInfo, data.ApiClient, db, data.Options.ConnectionId) + if err != nil { + return err + } + if isServerFlag { return nil } logger := taskCtx.GetLogger() - db := taskCtx.GetDal() apiCollector, err := api.NewStatefulApiCollector(api.RawDataSubTaskArgs{ Ctx: taskCtx, diff --git a/backend/plugins/jira/tasks/issue_changelog_extractor.go b/backend/plugins/jira/tasks/issue_changelog_extractor.go index a49142e3f..0cd9d4f65 100644 --- a/backend/plugins/jira/tasks/issue_changelog_extractor.go +++ b/backend/plugins/jira/tasks/issue_changelog_extractor.go @@ -23,7 +23,6 @@ import ( "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "github.com/apache/incubator-devlake/plugins/jira/models" "github.com/apache/incubator-devlake/plugins/jira/tasks/apiv2models" ) @@ -39,7 +38,12 @@ var ExtractIssueChangelogsMeta = plugin.SubTaskMeta{ func ExtractIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error { data := subtaskCtx.GetData().(*JiraTaskData) - if data.JiraServerInfo.DeploymentType == models.DeploymentServer { + + isServerFlag, err := isServer(data.JiraServerInfo, data.ApiClient, subtaskCtx.GetDal(), data.Options.ConnectionId) + if err != nil { + return err + } + if isServerFlag { return nil } connectionId := data.Options.ConnectionId diff --git a/backend/plugins/jira/tasks/issue_collector.go b/backend/plugins/jira/tasks/issue_collector.go index 84ac2d727..73f354969 100644 --- a/backend/plugins/jira/tasks/issue_collector.go +++ b/backend/plugins/jira/tasks/issue_collector.go @@ -172,7 +172,12 @@ func getTimeZone(taskCtx plugin.SubTaskContext) (*time.Location, errors.Error) { var resp *http.Response var path string var query url.Values - if data.JiraServerInfo.DeploymentType == models.DeploymentServer { + isServerFlag, err := isServer(data.JiraServerInfo, data.ApiClient, taskCtx.GetDal(), data.Options.ConnectionId) + if err != nil { + return nil, err + } + + if isServerFlag { path = "api/2/user" query = url.Values{"username": []string{conn.Username}} } else { diff --git a/backend/plugins/jira/tasks/issue_comment_collector.go b/backend/plugins/jira/tasks/issue_comment_collector.go index 5ef2777be..14bff5dd2 100644 --- a/backend/plugins/jira/tasks/issue_comment_collector.go +++ b/backend/plugins/jira/tasks/issue_comment_collector.go @@ -29,7 +29,6 @@ import ( "github.com/apache/incubator-devlake/core/log" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "github.com/apache/incubator-devlake/plugins/jira/models" "github.com/apache/incubator-devlake/plugins/jira/tasks/apiv2models" ) @@ -47,7 +46,11 @@ var CollectIssueCommentsMeta = plugin.SubTaskMeta{ func CollectIssueComments(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*JiraTaskData) - if data.JiraServerInfo.DeploymentType == models.DeploymentServer { + isServerFlag, err := isServer(data.JiraServerInfo, data.ApiClient, taskCtx.GetDal(), data.Options.ConnectionId) + if err != nil { + return err + } + if isServerFlag { return nil } logger := taskCtx.GetLogger() diff --git a/backend/plugins/jira/tasks/issue_comment_extractor.go b/backend/plugins/jira/tasks/issue_comment_extractor.go index e53828853..ada1a42ac 100644 --- a/backend/plugins/jira/tasks/issue_comment_extractor.go +++ b/backend/plugins/jira/tasks/issue_comment_extractor.go @@ -22,7 +22,6 @@ import ( "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "github.com/apache/incubator-devlake/plugins/jira/models" "github.com/apache/incubator-devlake/plugins/jira/tasks/apiv2models" ) @@ -38,7 +37,11 @@ var ExtractIssueCommentsMeta = plugin.SubTaskMeta{ func ExtractIssueComments(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*JiraTaskData) - if data.JiraServerInfo.DeploymentType == models.DeploymentServer { + isServerFlag, err := isServer(data.JiraServerInfo, data.ApiClient, taskCtx.GetDal(), data.Options.ConnectionId) + if err != nil { + return err + } + if isServerFlag { return nil } connectionId := data.Options.ConnectionId diff --git a/backend/plugins/jira/tasks/issue_type_collector.go b/backend/plugins/jira/tasks/issue_type_collector.go index c867316b7..5c4432d10 100644 --- a/backend/plugins/jira/tasks/issue_type_collector.go +++ b/backend/plugins/jira/tasks/issue_type_collector.go @@ -24,7 +24,6 @@ import ( "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "github.com/apache/incubator-devlake/plugins/jira/models" ) const RAW_ISSUE_TYPE_TABLE = "jira_api_issue_types" @@ -45,7 +44,11 @@ func CollectIssueTypes(taskCtx plugin.SubTaskContext) errors.Error { logger.Info("collect issue_types") urlTemplate := "api/3/issuetype" - if data.JiraServerInfo.DeploymentType == models.DeploymentServer { + isServerFlag, err := isServer(data.JiraServerInfo, data.ApiClient, taskCtx.GetDal(), data.Options.ConnectionId) + if err != nil { + return err + } + if isServerFlag { urlTemplate = "api/2/issuetype" } collector, err := api.NewApiCollector(api.ApiCollectorArgs{ diff --git a/backend/plugins/jira/tasks/shared.go b/backend/plugins/jira/tasks/shared.go index cdd118dd1..0fd0e5631 100644 --- a/backend/plugins/jira/tasks/shared.go +++ b/backend/plugins/jira/tasks/shared.go @@ -18,9 +18,11 @@ limitations under the License. package tasks import ( + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/jira/models" "net/http" ) @@ -46,3 +48,37 @@ func getStdStatus(statusKey string) string { return ticket.IN_PROGRESS } } + +func isServer(jiraServerInfo *models.JiraServerInfo, apiclient *api.ApiAsyncClient, db dal.Dal, connectionID uint64) (bool, errors.Error) { + if jiraServerInfo != nil { + return jiraServerInfo.IsDeploymentServer(), nil + } + // try to fetch jiraServerInfo from remote api + if apiclient != nil { + info, code, err := GetJiraServerInfo(apiclient) + if err != nil || code != http.StatusOK || info == nil { + return false, errors.HttpStatus(code).Wrap(err, "fail to get Jira server info") + } + return info.IsDeploymentServer(), nil + } + // fetch from db + info, err := getJiraServerInfoFromDB(db, connectionID) + if err != nil { + return false, err + } + if info == nil { + return false, nil + } + return info.IsDeploymentServer(), nil +} + +func getJiraServerInfoFromDB(db dal.Dal, connectionID uint64) (*models.JiraServerInfo, errors.Error) { + var info models.JiraServerInfo + if err := db.First(&info, dal.Where("connection_id = ?", connectionID)); err != nil { + if db.IsErrorNotFound(err) { + return nil, nil + } + return nil, err + } + return &info, nil +} diff --git a/backend/plugins/jira/tasks/sprint_extractor.go b/backend/plugins/jira/tasks/sprint_extractor.go index 21276353d..b1012dcec 100644 --- a/backend/plugins/jira/tasks/sprint_extractor.go +++ b/backend/plugins/jira/tasks/sprint_extractor.go @@ -39,9 +39,10 @@ var ExtractSprintsMeta = plugin.SubTaskMeta{ func ExtractSprints(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*JiraTaskData) - isServer := false - if data.JiraServerInfo.DeploymentType == models.DeploymentServer { - isServer = true + + isServerFlag, err := isServer(data.JiraServerInfo, data.ApiClient, taskCtx.GetDal(), data.Options.ConnectionId) + if err != nil { + return err } extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ @@ -64,7 +65,7 @@ func ExtractSprints(taskCtx plugin.SubTaskContext) errors.Error { BoardId: data.Options.BoardId, SprintId: sprint.ID, } - return []interface{}{sprint.ToToolLayer(data.Options.ConnectionId, isServer), &boardSprint}, nil + return []interface{}{sprint.ToToolLayer(data.Options.ConnectionId, isServerFlag), &boardSprint}, nil }, }) diff --git a/backend/plugins/jira/tasks/task_data.go b/backend/plugins/jira/tasks/task_data.go index 1b0580396..2f9c12e86 100644 --- a/backend/plugins/jira/tasks/task_data.go +++ b/backend/plugins/jira/tasks/task_data.go @@ -36,7 +36,7 @@ type JiraOptions struct { type JiraTaskData struct { Options *JiraOptions ApiClient *api.ApiAsyncClient - JiraServerInfo models.JiraServerInfo + JiraServerInfo *models.JiraServerInfo } type JiraApiParams models.JiraApiParams diff --git a/backend/plugins/opsgenie/impl/impl.go b/backend/plugins/opsgenie/impl/impl.go index fc234e7aa..5785d53ff 100644 --- a/backend/plugins/opsgenie/impl/impl.go +++ b/backend/plugins/opsgenie/impl/impl.go @@ -123,15 +123,20 @@ func (p Opsgenie) PrepareTaskData(taskCtx plugin.TaskContext, options map[string return nil, errors.Default.Wrap(err, "unable to get Opsgenie connection by the given connection ID") } - client, err := helper.NewApiClientFromConnection(taskCtx.GetContext(), taskCtx, connection) - - if err != nil { - return nil, err - } - asyncClient, err := helper.CreateAsyncApiClient(taskCtx, client, nil) - if err != nil { - return nil, err + var asyncClient *helper.ApiAsyncClient + syncPolicy := taskCtx.SyncPolicy() + if !syncPolicy.SkipCollectors { + client, err := helper.NewApiClientFromConnection(taskCtx.GetContext(), taskCtx, connection) + if err != nil { + return nil, err + } + newAsyncClient, err := helper.CreateAsyncApiClient(taskCtx, client, nil) + if err != nil { + return nil, err + } + asyncClient = newAsyncClient } + return &tasks.OpsgenieTaskData{ Options: op, Client: asyncClient, diff --git a/backend/plugins/pagerduty/impl/impl.go b/backend/plugins/pagerduty/impl/impl.go index 8ed6b21b9..b237b30e4 100644 --- a/backend/plugins/pagerduty/impl/impl.go +++ b/backend/plugins/pagerduty/impl/impl.go @@ -109,14 +109,19 @@ func (p PagerDuty) PrepareTaskData(taskCtx plugin.TaskContext, options map[strin return nil, errors.Default.Wrap(err, "unable to get Pagerduty connection by the given connection ID") } - client, err := helper.NewApiClientFromConnection(taskCtx.GetContext(), taskCtx, connection) - - if err != nil { - return nil, err - } - asyncClient, err := helper.CreateAsyncApiClient(taskCtx, client, nil) - if err != nil { - return nil, err + var asyncClient *helper.ApiAsyncClient + syncPolicy := taskCtx.SyncPolicy() + if !syncPolicy.SkipCollectors { + client, err := helper.NewApiClientFromConnection(taskCtx.GetContext(), taskCtx, connection) + + if err != nil { + return nil, err + } + newAsyncClient, err := helper.CreateAsyncApiClient(taskCtx, client, nil) + if err != nil { + return nil, err + } + asyncClient = newAsyncClient } return &tasks.PagerDutyTaskData{ Options: op, diff --git a/backend/plugins/slack/impl/impl.go b/backend/plugins/slack/impl/impl.go index 1b8e40940..8b3de9142 100644 --- a/backend/plugins/slack/impl/impl.go +++ b/backend/plugins/slack/impl/impl.go @@ -108,9 +108,14 @@ func (p Slack) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]in return nil, err } - apiClient, err := tasks.NewSlackApiClient(taskCtx, connection) - if err != nil { - return nil, err + var apiClient *helper.ApiAsyncClient + syncPolicy := taskCtx.SyncPolicy() + if !syncPolicy.SkipCollectors { + newApiClient, err := tasks.NewSlackApiClient(taskCtx, connection) + if err != nil { + return nil, err + } + apiClient = newApiClient } return &tasks.SlackTaskData{ Options: &op, diff --git a/backend/plugins/sonarqube/api/blueprint_v200.go b/backend/plugins/sonarqube/api/blueprint_v200.go index e2995aa65..b129b347b 100644 --- a/backend/plugins/sonarqube/api/blueprint_v200.go +++ b/backend/plugins/sonarqube/api/blueprint_v200.go @@ -51,11 +51,13 @@ func MakeDataSourcePipelinePlanV200( return nil, nil, err } - // needed for the connection to populate its access tokens - // if AppKey authentication method is selected - _, err = helper.NewApiClientFromConnection(context.TODO(), basicRes, connection) - if err != nil { - return nil, nil, err + if !skipCollectors { + // needed for the connection to populate its access tokens + // if AppKey authentication method is selected + _, err = helper.NewApiClientFromConnection(context.TODO(), basicRes, connection) + if err != nil { + return nil, nil, err + } } plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails, connection) diff --git a/backend/plugins/sonarqube/impl/impl.go b/backend/plugins/sonarqube/impl/impl.go index a364435b1..b6d214032 100644 --- a/backend/plugins/sonarqube/impl/impl.go +++ b/backend/plugins/sonarqube/impl/impl.go @@ -126,29 +126,37 @@ func (p Sonarqube) PrepareTaskData(taskCtx plugin.TaskContext, options map[strin return nil, errors.Default.Wrap(err, "unable to get Sonarqube connection by the given connection ID") } - apiClient, err := tasks.CreateApiClient(taskCtx, connection) - if err != nil { - return nil, errors.Default.Wrap(err, "unable to get Sonarqube API client instance") + var apiClient *helper.ApiAsyncClient + syncPolicy := taskCtx.SyncPolicy() + if !syncPolicy.SkipCollectors { + newApiClient, err := tasks.CreateApiClient(taskCtx, connection) + if err != nil { + return nil, errors.Default.Wrap(err, "unable to get Sonarqube API client instance") + } + apiClient = newApiClient } taskData := &tasks.SonarqubeTaskData{ Options: op, ApiClient: apiClient, TaskStartTime: time.Now(), } - // even we have project in _tool_sonaqube_projects, we still need to collect project to update LastAnalysisDate - var apiProject *models.SonarqubeApiProject - apiProject, err = api.GetApiProject(op.ProjectKey, apiClient) - if err != nil { - return nil, err - } - logger.Debug(fmt.Sprintf("Current project: %s", apiProject.ProjectKey)) - scope := apiProject.ConvertApiScope() - scope.ConnectionId = op.ConnectionId - err = taskCtx.GetDal().CreateOrUpdate(&scope) - if err != nil { - return nil, err + if apiClient != nil { + // even we have project in _tool_sonarqube_projects, we still need to collect project to update LastAnalysisDate + var apiProject *models.SonarqubeApiProject + apiProject, err = api.GetApiProject(op.ProjectKey, apiClient) + if err != nil { + return nil, err + } + logger.Debug(fmt.Sprintf("Current project: %s", apiProject.ProjectKey)) + scope := apiProject.ConvertApiScope() + scope.ConnectionId = op.ConnectionId + + err = taskCtx.GetDal().CreateOrUpdate(&scope) + if err != nil { + return nil, err + } + taskData.LastAnalysisDate = scope.LastAnalysisDate.ToNullableTime() } - taskData.LastAnalysisDate = scope.LastAnalysisDate.ToNullableTime() return taskData, nil } diff --git a/backend/plugins/tapd/impl/impl.go b/backend/plugins/tapd/impl/impl.go index 0cd823601..2885a4337 100644 --- a/backend/plugins/tapd/impl/impl.go +++ b/backend/plugins/tapd/impl/impl.go @@ -204,9 +204,14 @@ func (p Tapd) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int if connection.RateLimitPerHour == 0 { connection.RateLimitPerHour = 3600 } - tapdApiClient, err := tasks.NewTapdApiClient(taskCtx, connection) - if err != nil { - return nil, errors.Default.Wrap(err, "failed to create tapd api client") + var apiClient *helper.ApiAsyncClient + syncPolicy := taskCtx.SyncPolicy() + if !syncPolicy.SkipCollectors { + tapdApiClient, err := tasks.NewTapdApiClient(taskCtx, connection) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to create tapd api client") + } + apiClient = tapdApiClient } if op.WorkspaceId != 0 { @@ -238,7 +243,7 @@ func (p Tapd) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int op.CstZone = cstZone taskData := &tasks.TapdTaskData{ Options: op, - ApiClient: tapdApiClient, + ApiClient: apiClient, Connection: connection, } return taskData, nil diff --git a/backend/plugins/teambition/impl/impl.go b/backend/plugins/teambition/impl/impl.go index d31a58736..662cace0f 100644 --- a/backend/plugins/teambition/impl/impl.go +++ b/backend/plugins/teambition/impl/impl.go @@ -142,9 +142,14 @@ func (p Teambition) PrepareTaskData(taskCtx plugin.TaskContext, options map[stri return nil, errors.Default.Wrap(err, "unable to get Teambition connection by the given connection ID") } - apiClient, err := tasks.NewTeambitionApiClient(taskCtx, connection) - if err != nil { - return nil, errors.Default.Wrap(err, "unable to get Teambition API client instance") + var apiClient *helper.ApiAsyncClient + syncPolicy := taskCtx.SyncPolicy() + if !syncPolicy.SkipCollectors { + newApiClient, err := tasks.NewTeambitionApiClient(taskCtx, connection) + if err != nil { + return nil, errors.Default.Wrap(err, "unable to get Teambition API client instance") + } + apiClient = newApiClient } taskData := &tasks.TeambitionTaskData{ Options: op, diff --git a/backend/plugins/trello/impl/impl.go b/backend/plugins/trello/impl/impl.go index 35adaa2c1..b3a0f088d 100644 --- a/backend/plugins/trello/impl/impl.go +++ b/backend/plugins/trello/impl/impl.go @@ -116,9 +116,14 @@ func (p Trello) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i if err != nil { return nil, errors.Default.Wrap(err, "error getting connection for Trello plugin") } - apiClient, err := tasks.CreateApiClient(taskCtx, connection) - if err != nil { - return nil, err + var apiClient *helper.ApiAsyncClient + syncPolicy := taskCtx.SyncPolicy() + if !syncPolicy.SkipCollectors { + newApiClient, err := tasks.CreateApiClient(taskCtx, connection) + if err != nil { + return nil, err + } + apiClient = newApiClient } return &tasks.TrelloTaskData{ Options: &op, diff --git a/backend/plugins/zentao/impl/impl.go b/backend/plugins/zentao/impl/impl.go index 4317fa574..5aa74c966 100644 --- a/backend/plugins/zentao/impl/impl.go +++ b/backend/plugins/zentao/impl/impl.go @@ -19,6 +19,8 @@ package impl import ( "fmt" + "net/url" + "strings" "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/dal" @@ -182,9 +184,14 @@ func (p Zentao) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i return nil, errors.Default.Wrap(err, "unable to get Zentao connection by the given connection ID: %v") } - apiClient, err := tasks.NewZentaoApiClient(taskCtx, connection) - if err != nil { - return nil, errors.Default.Wrap(err, "unable to get Zentao API client instance: %v") + var apiClient *helper.ApiAsyncClient + syncPolicy := taskCtx.SyncPolicy() + if !syncPolicy.SkipCollectors { + newApiClient, err := tasks.NewZentaoApiClient(taskCtx, connection) + if err != nil { + return nil, errors.Default.Wrap(err, "unable to get Zentao API client instance: %v") + } + apiClient = newApiClient } if op.ScopeConfig == nil && op.ScopeConfigId != 0 { @@ -203,36 +210,64 @@ func (p Zentao) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i AccountCache: tasks.NewAccountCache(taskCtx.GetDal(), op.ConnectionId), } - if connection.DbUrl != "" { - if connection.DbLoggingLevel == "" { - connection.DbLoggingLevel = taskCtx.GetConfig("DB_LOGGING_LEVEL") - } + if !syncPolicy.SkipCollectors { + if connection.DbUrl != "" { + if connection.DbLoggingLevel == "" { + connection.DbLoggingLevel = taskCtx.GetConfig("DB_LOGGING_LEVEL") + } - if connection.DbIdleConns == 0 { - connection.DbIdleConns = taskCtx.GetConfigReader().GetInt("DB_IDLE_CONNS") - } + if connection.DbIdleConns == 0 { + connection.DbIdleConns = taskCtx.GetConfigReader().GetInt("DB_IDLE_CONNS") + } - if connection.DbMaxConns == 0 { - connection.DbMaxConns = taskCtx.GetConfigReader().GetInt("DB_MAX_CONNS") - } + if connection.DbMaxConns == 0 { + connection.DbMaxConns = taskCtx.GetConfigReader().GetInt("DB_MAX_CONNS") + } - v := viper.New() - v.Set("DB_URL", connection.DbUrl) - v.Set("DB_LOGGING_LEVEL", connection.DbLoggingLevel) - v.Set("DB_IDLE_CONNS", connection.DbIdleConns) - v.Set("DbMaxConns", connection.DbMaxConns) + v := viper.New() + v.Set("DB_URL", connection.DbUrl) + v.Set("DB_LOGGING_LEVEL", connection.DbLoggingLevel) + v.Set("DB_IDLE_CONNS", connection.DbIdleConns) + v.Set("DbMaxConns", connection.DbMaxConns) - rgorm, err := runner.NewGormDb(v, taskCtx.GetLogger()) - if err != nil { - return nil, errors.Default.Wrap(err, fmt.Sprintf("failed to connect to the zentao remote databases %s", connection.DbUrl)) + rgorm, err := runner.NewGormDb(v, taskCtx.GetLogger()) + if err != nil { + return nil, errors.Default.Wrap(err, fmt.Sprintf("failed to connect to the zentao remote databases %s", connection.DbUrl)) + } + + data.RemoteDb = dalgorm.NewDalgorm(rgorm) } + } - data.RemoteDb = dalgorm.NewDalgorm(rgorm) + endpoint := connection.Endpoint + if data.ApiClient != nil { + endpoint = data.ApiClient.GetEndpoint() + } + homepage, err := getZentaoHomePage(endpoint) + if err != nil { + return data, errors.Convert(err) } + data.HomePageURL = homepage return data, nil } +// getZentaoHomePage receive endpoint like "http://54.158.1.10:30001/api.php/v1/"; and return zentao's homepage like "http://54.158.1.10:30001/"; +func getZentaoHomePage(endpoint string) (string, error) { + if endpoint == "" { + return "", errors.Default.New("empty endpoint") + } + endpointURL, err := url.Parse(endpoint) + if err != nil { + return "", err + } else { + protocol := endpointURL.Scheme + host := endpointURL.Host + zentaoPath, _, _ := strings.Cut(endpointURL.Path, "/api.php/v1") + return fmt.Sprintf("%s://%s%s", protocol, host, zentaoPath), nil + } +} + // RootPkgPath information lost when compiled as plugin(.so) func (p Zentao) RootPkgPath() string { return "github.com/apache/incubator-devlake/plugins/zentao" diff --git a/backend/plugins/zentao/tasks/execution_convertor.go b/backend/plugins/zentao/tasks/execution_convertor.go index 41d80d3bb..8fda2676e 100644 --- a/backend/plugins/zentao/tasks/execution_convertor.go +++ b/backend/plugins/zentao/tasks/execution_convertor.go @@ -44,7 +44,6 @@ var ConvertExecutionMeta = plugin.SubTaskMeta{ func ConvertExecutions(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*ZentaoTaskData) db := taskCtx.GetDal() - logger := taskCtx.GetLogger() executionIdGen := didgen.NewDomainIdGenerator(&models.ZentaoExecution{}) projectIdGen := didgen.NewDomainIdGenerator(&models.ZentaoProject{}) cursor, err := db.Cursor( @@ -54,12 +53,6 @@ func ConvertExecutions(taskCtx plugin.SubTaskContext) errors.Error { if err != nil { return err } - - homePage, getZentaoHomePageErr := getZentaoHomePage(data.ApiClient.GetEndpoint()) - if getZentaoHomePageErr != nil { - logger.Error(getZentaoHomePageErr, "get zentao homepage") - return errors.Default.WrapRaw(getZentaoHomePageErr) - } defer cursor.Close() convertor, err := api.NewDataConverter(api.DataConverterArgs{ InputRowType: reflect.TypeOf(models.ZentaoExecution{}), @@ -91,7 +84,7 @@ func ConvertExecutions(taskCtx plugin.SubTaskContext) errors.Error { Id: executionIdGen.Generate(toolExecution.ConnectionId, toolExecution.Id), }, Name: toolExecution.Name, - Url: fmt.Sprintf("%s/execution-view-%d.html", homePage, toolExecution.Id), + Url: fmt.Sprintf("%s/execution-view-%d.html", data.HomePageURL, toolExecution.Id), Status: domainStatus, StartedDate: toolExecution.RealBegan.ToNullableTime(), EndedDate: toolExecution.PlanEnd.ToNullableTime(), diff --git a/backend/plugins/zentao/tasks/project_convertor.go b/backend/plugins/zentao/tasks/project_convertor.go index b09ef9405..d45a00758 100644 --- a/backend/plugins/zentao/tasks/project_convertor.go +++ b/backend/plugins/zentao/tasks/project_convertor.go @@ -45,7 +45,6 @@ var ConvertProjectMeta = plugin.SubTaskMeta{ func ConvertProjects(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*ZentaoTaskData) db := taskCtx.GetDal() - logger := taskCtx.GetLogger() boardIdGen := didgen.NewDomainIdGenerator(&models.ZentaoProject{}) cursor, err := db.Cursor( dal.From(&models.ZentaoProject{}), @@ -55,11 +54,6 @@ func ConvertProjects(taskCtx plugin.SubTaskContext) errors.Error { return err } defer cursor.Close() - homePage, getZentaoHomePageErr := getZentaoHomePage(data.ApiClient.GetEndpoint()) - if getZentaoHomePageErr != nil { - logger.Error(getZentaoHomePageErr, "get zentao homepage") - return errors.Default.WrapRaw(getZentaoHomePageErr) - } convertor, err := api.NewDataConverter(api.DataConverterArgs{ InputRowType: reflect.TypeOf(models.ZentaoProject{}), Input: cursor, @@ -79,7 +73,7 @@ func ConvertProjects(taskCtx plugin.SubTaskContext) errors.Error { Description: toolProject.Description, CreatedDate: toolProject.OpenedDate.ToNullableTime(), Type: "scrum", - Url: fmt.Sprintf("%s/project-index-%d.html", homePage, data.Options.ProjectId), + Url: fmt.Sprintf("%s/project-index-%d.html", data.HomePageURL, data.Options.ProjectId), } results := make([]interface{}, 0) results = append(results, domainBoard) diff --git a/backend/plugins/zentao/tasks/shared.go b/backend/plugins/zentao/tasks/shared.go index aa1140573..de4de2307 100644 --- a/backend/plugins/zentao/tasks/shared.go +++ b/backend/plugins/zentao/tasks/shared.go @@ -339,19 +339,3 @@ func extractIdFromLogComment(logCommentType string, comment string) ([]string, e } return ret, nil } - -// getZentaoHomePage receive endpoint like "http://54.158.1.10:30001/api.php/v1/"; and return zentao's homepage like "http://54.158.1.10:30001/"; -func getZentaoHomePage(endpoint string) (string, error) { - if endpoint == "" { - return "", errors.Default.New("empty endpoint") - } - endpointURL, err := url.Parse(endpoint) - if err != nil { - return "", err - } else { - protocol := endpointURL.Scheme - host := endpointURL.Host - zentaoPath, _, _ := strings.Cut(endpointURL.Path, "/api.php/v1") - return fmt.Sprintf("%s://%s%s", protocol, host, zentaoPath), nil - } -} diff --git a/backend/plugins/zentao/tasks/task_data.go b/backend/plugins/zentao/tasks/task_data.go index 26bc9734c..a6a097f21 100644 --- a/backend/plugins/zentao/tasks/task_data.go +++ b/backend/plugins/zentao/tasks/task_data.go @@ -57,6 +57,7 @@ type ZentaoTaskData struct { Bugs map[int64]struct{} AccountCache *AccountCache ApiClient *helper.ApiAsyncClient + HomePageURL string } func DecodeAndValidateTaskOptions(options map[string]interface{}) (*ZentaoOptions, error) {
