This is an automated email from the ASF dual-hosted git repository. abeizn pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit ed07e6b88b5d90d2b125671ac83543ce291a5e27 Author: Yingchu Chen <[email protected]> AuthorDate: Wed Oct 12 19:38:54 2022 +0800 fix(github): modify blueprint --- plugins/github/api/blueprint.go | 251 ++++++++++++++++++++-------------------- 1 file changed, 124 insertions(+), 127 deletions(-) diff --git a/plugins/github/api/blueprint.go b/plugins/github/api/blueprint.go index 48ec1a1f..8acb4d7f 100644 --- a/plugins/github/api/blueprint.go +++ b/plugins/github/api/blueprint.go @@ -38,117 +38,87 @@ import ( func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV100) (core.PipelinePlan, errors.Error) { var err errors.Error - plan := make(core.PipelinePlan, len(scope)) - for i, scopeElem := range scope { - plan, err = processScope(subtaskMetas, connectionId, scopeElem, i, plan, nil, nil) - if err != nil { - return nil, err - } - } - return plan, nil -} -func processScope(subtaskMetas []core.SubTaskMeta, connectionId uint64, scopeElem *core.BlueprintScopeV100, i int, plan core.PipelinePlan, apiRepo *tasks.GithubApiRepo, connection *models.GithubConnection) (core.PipelinePlan, errors.Error) { - var err errors.Error - // handle taskOptions and transformationRules, by dumping them to taskOptions - transformationRules := make(map[string]interface{}) - if len(scopeElem.Transformation) > 0 { - err = errors.Convert(json.Unmarshal(scopeElem.Transformation, &transformationRules)) - if err != nil { - return nil, err - } - } - // refdiff - if refdiffRules, ok := transformationRules["refdiff"]; ok && refdiffRules != nil { - // add a new task to next stage - j := i + 1 - if j == len(plan) { - plan = append(plan, nil) - } - plan[j] = core.PipelineStage{ - { - Plugin: "refdiff", - Options: refdiffRules.(map[string]interface{}), - }, - } - // remove it from github transformationRules - delete(transformationRules, "refdiff") - } - // construct task options for github - options := make(map[string]interface{}) - err = errors.Convert(json.Unmarshal(scopeElem.Options, &options)) + connection := new(models.GithubConnection) + err = connectionHelper.FirstById(connection, connectionId) if err != nil { return nil, err } - options["connectionId"] = connectionId - options["transformationRules"] = transformationRules - // make sure task options is valid - op, err := tasks.DecodeAndValidateTaskOptions(options) - if err != nil { - return nil, err - } - // construct subtasks - subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, scopeElem.Entities) + token := strings.Split(connection.Token, ",")[0] + + apiClient, err := helper.NewApiClient( + context.TODO(), + connection.Endpoint, + map[string]string{ + "Authorization": fmt.Sprintf("Bearer %s", token), + }, + 10*time.Second, + connection.Proxy, + basicRes, + ) + plan, err := makePipelinePlan(subtaskMetas, connectionId, scope, apiClient, connection) if err != nil { return nil, err } - stage := plan[i] - if stage == nil { - stage = core.PipelineStage{} - } - stage = append(stage, &core.PipelineTask{ - Plugin: "github", - Subtasks: subtasks, - Options: options, - }) - // collect git data by gitextractor if CODE was requested - if utils.StringsContains(scopeElem.Entities, core.DOMAIN_TYPE_CODE) { - // here is the tricky part, we have to obtain the repo id beforehand - if connection == nil { - connection = new(models.GithubConnection) - err = connectionHelper.FirstById(connection, connectionId) + return plan, nil +} +func makePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV100, apiClient *helper.ApiClient, connection *models.GithubConnection) (core.PipelinePlan, errors.Error) { + var err errors.Error + plan := make(core.PipelinePlan, len(scope)) + for i, scopeElem := range scope { + // handle taskOptions and transformationRules, by dumping them to taskOptions + transformationRules := make(map[string]interface{}) + if len(scopeElem.Transformation) > 0 { + err = errors.Convert(json.Unmarshal(scopeElem.Transformation, &transformationRules)) if err != nil { return nil, err } } - token := strings.Split(connection.Token, ",")[0] - if apiRepo == nil { - apiRepo = new(tasks.GithubApiRepo) - err = getApiRepo(connection, token, op, apiRepo) - if err != nil { - return nil, err + // refdiff + if refdiffRules, ok := transformationRules["refdiff"]; ok && refdiffRules != nil { + // add a new task to next stage + j := i + 1 + if j == len(plan) { + plan = append(plan, nil) + } + plan[j] = core.PipelineStage{ + { + Plugin: "refdiff", + Options: refdiffRules.(map[string]interface{}), + }, } + // remove it from github transformationRules + delete(transformationRules, "refdiff") } - cloneUrl, err := errors.Convert01(url.Parse(apiRepo.CloneUrl)) + // construct task options for github + options := make(map[string]interface{}) + err = errors.Convert(json.Unmarshal(scopeElem.Options, &options)) if err != nil { return nil, err } - cloneUrl.User = url.UserPassword("git", token) - stage = append(stage, &core.PipelineTask{ - Plugin: "gitextractor", - Options: map[string]interface{}{ - "url": cloneUrl.String(), - "repoId": didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connectionId, apiRepo.GithubId), - "proxy": connection.Proxy, - }, - }) - } - // dora - if productionPattern, ok := transformationRules["productionPattern"]; ok && productionPattern != nil { - j := i + 1 - if j == len(plan) { - plan = append(plan, nil) - } - // add a new task to next stage - if plan[j] != nil { - j++ - } - if j == len(plan) { - plan = append(plan, nil) + options["connectionId"] = connectionId + options["transformationRules"] = transformationRules + // make sure task options is valid + op, err := tasks.DecodeAndValidateTaskOptions(options) + if err != nil { + return nil, err } + // construct subtasks + subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, scopeElem.Entities) if err != nil { return nil, err } - if apiRepo == nil { + stage := plan[i] + if stage == nil { + stage = core.PipelineStage{} + } + stage = append(stage, &core.PipelineTask{ + Plugin: "github", + Subtasks: subtasks, + Options: options, + }) + // collect git data by gitextractor if CODE was requested + if utils.StringsContains(scopeElem.Entities, core.DOMAIN_TYPE_CODE) { + // here is the tricky part, we have to obtain the repo id beforehand if connection == nil { connection = new(models.GithubConnection) err = connectionHelper.FirstById(connection, connectionId) @@ -157,60 +127,87 @@ func processScope(subtaskMetas []core.SubTaskMeta, connectionId uint64, scopeEle } } token := strings.Split(connection.Token, ",")[0] - apiRepo = new(tasks.GithubApiRepo) - err = getApiRepo(connection, token, op, apiRepo) + apiRepo, err := getApiRepo(op, apiClient) + if err != nil { + return nil, err + } + + cloneUrl, err := errors.Convert01(url.Parse(apiRepo.CloneUrl)) if err != nil { return nil, err } + cloneUrl.User = url.UserPassword("git", token) + stage = append(stage, &core.PipelineTask{ + Plugin: "gitextractor", + Options: map[string]interface{}{ + "url": cloneUrl.String(), + "repoId": didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connectionId, apiRepo.GithubId), + "proxy": connection.Proxy, + }, + }) } - doraOption := make(map[string]interface{}) - doraOption["repoId"] = didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connectionId, apiRepo.GithubId) - doraRules := make(map[string]interface{}) - doraRules["productionPattern"] = productionPattern - doraOption["transformationRules"] = doraRules - plan[j] = core.PipelineStage{ - { - Plugin: "dora", - Subtasks: []string{"EnrichTaskEnv"}, - Options: doraOption, - }, + // dora + if productionPattern, ok := transformationRules["productionPattern"]; ok && productionPattern != nil { + j := i + 1 + if j == len(plan) { + plan = append(plan, nil) + } + // add a new task to next stage + if plan[j] != nil { + j++ + } + if j == len(plan) { + plan = append(plan, nil) + } + if err != nil { + return nil, err + } + + apiRepo, err := getApiRepo(op, apiClient) + if err != nil { + return nil, err + } + + doraOption := make(map[string]interface{}) + doraOption["repoId"] = didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connectionId, apiRepo.GithubId) + doraRules := make(map[string]interface{}) + doraRules["productionPattern"] = productionPattern + doraOption["transformationRules"] = doraRules + plan[j] = core.PipelineStage{ + { + Plugin: "dora", + Subtasks: []string{"EnrichTaskEnv"}, + Options: doraOption, + }, + } + // remove it from github transformationRules + delete(transformationRules, "productionPattern") + } + plan[i] = stage + if err != nil { + return nil, err } - // remove it from github transformationRules - delete(transformationRules, "productionPattern") } - plan[i] = stage return plan, nil } -func getApiRepo(connection *models.GithubConnection, token string, op *tasks.GithubOptions, apiRepo *tasks.GithubApiRepo) errors.Error { - apiClient, err := helper.NewApiClient( - context.TODO(), - connection.Endpoint, - map[string]string{ - "Authorization": fmt.Sprintf("Bearer %s", token), - }, - 10*time.Second, - connection.Proxy, - basicRes, - ) - if err != nil { - return err - } +func getApiRepo(op *tasks.GithubOptions, apiClient *helper.ApiClient) (*tasks.GithubApiRepo, errors.Error) { + apiRepo := new(tasks.GithubApiRepo) res, err := apiClient.Get(fmt.Sprintf("repos/%s/%s", op.Owner, op.Repo), nil, nil) if err != nil { - return err + return nil, err } //defer res.Body.Close() if res.StatusCode != http.StatusOK { - return errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code when requesting repo detail from %s", res.Request.URL.String())) + return nil, errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code when requesting repo detail from %s", res.Request.URL.String())) } body, err := errors.Convert01(io.ReadAll(res.Body)) if err != nil { - return err + return nil, err } err = errors.Convert(json.Unmarshal(body, apiRepo)) if err != nil { - return err + return nil, err } - return nil + return apiRepo, nil }
