This is an automated email from the ASF dual-hosted git repository. abeizn pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 2d201b96ff061a1713be0363f49a19bfe2f89f4c Author: abeizn <[email protected]> AuthorDate: Thu Jun 23 22:54:18 2022 +0800 feat: jenkins blueprint normal mode support --- plugins/jenkins/api/blueprint.go | 59 ++++++++++++++++++++++++++++++++ plugins/jenkins/impl/impl.go | 12 ++++--- plugins/jenkins/tasks/build_collector.go | 1 + plugins/jenkins/tasks/build_convertor.go | 1 + plugins/jenkins/tasks/build_extractor.go | 1 + plugins/jenkins/tasks/job_collector.go | 1 + plugins/jenkins/tasks/job_convertor.go | 1 + plugins/jenkins/tasks/job_extractor.go | 1 + plugins/jenkins/tasks/task_data.go | 15 ++++++++ 9 files changed, 87 insertions(+), 5 deletions(-) diff --git a/plugins/jenkins/api/blueprint.go b/plugins/jenkins/api/blueprint.go new file mode 100644 index 00000000..40b56357 --- /dev/null +++ b/plugins/jenkins/api/blueprint.go @@ -0,0 +1,59 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "encoding/json" + "fmt" + + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/helper" + "github.com/apache/incubator-devlake/plugins/jenkins/tasks" +) + +func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV100) (core.PipelinePlan, error) { + var err error + plan := make(core.PipelinePlan, len(scope)) + for i, scopeElem := range scope { + // handle taskOptions and transformationRules, by dumping them to taskOptions + taskOptions := make(map[string]interface{}) + err = json.Unmarshal(scopeElem.Options, &taskOptions) + if err != nil { + fmt.Println("1111") + return nil, err + } + taskOptions["connectionId"] = connectionId + _, err := tasks.DecodeAndValidateTaskOptions(taskOptions) + if err != nil { + return nil, err + } + // subtasks + subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, scopeElem.Entities) + if err != nil { + return nil, err + } + stage := core.PipelineStage{ + { + Plugin: "jenkins", + Subtasks: subtasks, + Options: taskOptions, + }, + } + + plan[i] = stage + } + return plan, nil +} diff --git a/plugins/jenkins/impl/impl.go b/plugins/jenkins/impl/impl.go index c7bf3d89..bb0885f5 100644 --- a/plugins/jenkins/impl/impl.go +++ b/plugins/jenkins/impl/impl.go @@ -27,7 +27,6 @@ import ( "github.com/apache/incubator-devlake/plugins/jenkins/models" "github.com/apache/incubator-devlake/plugins/jenkins/models/migrationscripts" "github.com/apache/incubator-devlake/plugins/jenkins/tasks" - "github.com/mitchellh/mapstructure" "github.com/spf13/viper" "gorm.io/gorm" ) @@ -60,10 +59,9 @@ func (plugin Jenkins) SubTaskMetas() []core.SubTaskMeta { } } func (plugin Jenkins) PrepareTaskData(taskCtx core.TaskContext, options map[string]interface{}) (interface{}, error) { - var op tasks.JenkinsOptions - err := mapstructure.Decode(options, &op) + op, err := tasks.DecodeAndValidateTaskOptions(options) if err != nil { - return nil, fmt.Errorf("Failed to decode options: %v", err) + return nil, err } if op.ConnectionId == 0 { return nil, fmt.Errorf("connectionId is invalid") @@ -87,7 +85,7 @@ func (plugin Jenkins) PrepareTaskData(taskCtx core.TaskContext, options map[stri return nil, err } return &tasks.JenkinsTaskData{ - Options: &op, + Options: op, ApiClient: apiClient, Connection: connection, }, nil @@ -103,6 +101,10 @@ func (plugin Jenkins) MigrationScripts() []migration.Script { } } +func (plugin Jenkins) MakePipelinePlan(connectionId uint64, scope []*core.BlueprintScopeV100) (core.PipelinePlan, error) { + return api.MakePipelinePlan(plugin.SubTaskMetas(), connectionId, scope) +} + func (plugin Jenkins) ApiResources() map[string]map[string]core.ApiResourceHandler { return map[string]map[string]core.ApiResourceHandler{ "test": { diff --git a/plugins/jenkins/tasks/build_collector.go b/plugins/jenkins/tasks/build_collector.go index 074cd276..85c8f2ad 100644 --- a/plugins/jenkins/tasks/build_collector.go +++ b/plugins/jenkins/tasks/build_collector.go @@ -36,6 +36,7 @@ var CollectApiBuildsMeta = core.SubTaskMeta{ EntryPoint: CollectApiBuilds, EnabledByDefault: true, Description: "Collect builds data from jenkins api", + DomainTypes: []string{core.DOMAIN_TYPE_CICD}, } type SimpleJob struct { diff --git a/plugins/jenkins/tasks/build_convertor.go b/plugins/jenkins/tasks/build_convertor.go index 12d4a6a2..9d3cb941 100644 --- a/plugins/jenkins/tasks/build_convertor.go +++ b/plugins/jenkins/tasks/build_convertor.go @@ -34,6 +34,7 @@ var ConvertBuildsMeta = core.SubTaskMeta{ EntryPoint: ConvertBuilds, EnabledByDefault: true, Description: "Convert tool layer table jenkins_builds into domain layer table builds", + DomainTypes: []string{core.DOMAIN_TYPE_CICD}, } func ConvertBuilds(taskCtx core.SubTaskContext) error { diff --git a/plugins/jenkins/tasks/build_extractor.go b/plugins/jenkins/tasks/build_extractor.go index a3df36fb..bfc8e2ed 100644 --- a/plugins/jenkins/tasks/build_extractor.go +++ b/plugins/jenkins/tasks/build_extractor.go @@ -34,6 +34,7 @@ var ExtractApiBuildsMeta = core.SubTaskMeta{ EntryPoint: ExtractApiBuilds, EnabledByDefault: true, Description: "Extract raw builds data into tool layer table jenkins_builds", + DomainTypes: []string{core.DOMAIN_TYPE_CICD}, } func ExtractApiBuilds(taskCtx core.SubTaskContext) error { diff --git a/plugins/jenkins/tasks/job_collector.go b/plugins/jenkins/tasks/job_collector.go index ad4d32bd..d010706c 100644 --- a/plugins/jenkins/tasks/job_collector.go +++ b/plugins/jenkins/tasks/job_collector.go @@ -34,6 +34,7 @@ var CollectApiJobsMeta = core.SubTaskMeta{ EntryPoint: CollectApiJobs, EnabledByDefault: true, Description: "Collect jobs data from jenkins api", + DomainTypes: []string{core.DOMAIN_TYPE_CICD}, } func CollectApiJobs(taskCtx core.SubTaskContext) error { diff --git a/plugins/jenkins/tasks/job_convertor.go b/plugins/jenkins/tasks/job_convertor.go index 11e1bb8b..e9117ad7 100644 --- a/plugins/jenkins/tasks/job_convertor.go +++ b/plugins/jenkins/tasks/job_convertor.go @@ -34,6 +34,7 @@ var ConvertJobsMeta = core.SubTaskMeta{ EntryPoint: ConvertJobs, EnabledByDefault: true, Description: "Convert tool layer table jenkins_jobs into domain layer table jobs", + DomainTypes: []string{core.DOMAIN_TYPE_CICD}, } func ConvertJobs(taskCtx core.SubTaskContext) error { diff --git a/plugins/jenkins/tasks/job_extractor.go b/plugins/jenkins/tasks/job_extractor.go index 18967eea..6f289008 100644 --- a/plugins/jenkins/tasks/job_extractor.go +++ b/plugins/jenkins/tasks/job_extractor.go @@ -32,6 +32,7 @@ var ExtractApiJobsMeta = core.SubTaskMeta{ EntryPoint: ExtractApiJobs, EnabledByDefault: true, Description: "Extract raw jobs data into tool layer table jenkins_jobs", + DomainTypes: []string{core.DOMAIN_TYPE_CICD}, } func ExtractApiJobs(taskCtx core.SubTaskContext) error { diff --git a/plugins/jenkins/tasks/task_data.go b/plugins/jenkins/tasks/task_data.go index 1ee92532..b0d35c6f 100644 --- a/plugins/jenkins/tasks/task_data.go +++ b/plugins/jenkins/tasks/task_data.go @@ -18,10 +18,12 @@ limitations under the License. package tasks import ( + "fmt" "time" "github.com/apache/incubator-devlake/plugins/helper" "github.com/apache/incubator-devlake/plugins/jenkins/models" + "github.com/mitchellh/mapstructure" ) type JenkinsApiParams struct { @@ -40,3 +42,16 @@ type JenkinsTaskData struct { Connection *models.JenkinsConnection Since *time.Time } + +func DecodeAndValidateTaskOptions(options map[string]interface{}) (*JenkinsOptions, error) { + var op JenkinsOptions + err := mapstructure.Decode(options, &op) + if err != nil { + return nil, err + } + // find the needed Jenkins now + if op.ConnectionId == 0 { + return nil, fmt.Errorf("connectionId is invalid") + } + return &op, nil +}
