This is an automated email from the ASF dual-hosted git repository. warren pushed a commit to branch feat-plugin-zentao in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit e75bcd36c00a5d3736a13e400860b49065207038 Author: Yingchu Chen <[email protected]> AuthorDate: Tue Sep 6 18:03:04 2022 +0800 feat(zentao): create new plugin Relate to #2961 --- plugins/zentao/api/blueprint.go | 69 ++++++++++ plugins/zentao/api/connection.go | 149 +++++++++++++++++++++ plugins/zentao/api/init.go | 39 ++++++ plugins/zentao/impl/impl.go | 124 +++++++++++++++++ plugins/zentao/models/archived/connection.go | 70 ++++++++++ plugins/zentao/models/connection.go | 51 +++++++ .../migrationscripts/20220906_add_init_tables.go | 39 ++++++ plugins/zentao/models/migrationscripts/register.go | 27 ++++ plugins/zentao/tasks/api_client.go | 92 +++++++++++++ plugins/zentao/tasks/project_collector.go | 78 +++++++++++ plugins/zentao/tasks/task_data.go | 61 +++++++++ plugins/zentao/zentao.go | 47 +++++++ 12 files changed, 846 insertions(+) diff --git a/plugins/zentao/api/blueprint.go b/plugins/zentao/api/blueprint.go new file mode 100644 index 00000000..3fa0e25f --- /dev/null +++ b/plugins/zentao/api/blueprint.go @@ -0,0 +1,69 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/helper" + "github.com/apache/incubator-devlake/plugins/zentao/tasks" +) + +func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV100) (core.PipelinePlan, error) { + var err error + plan := make(core.PipelinePlan, len(scope)) + for i, scopeElem := range scope { + taskOptions := make(map[string]interface{}) + err = json.Unmarshal(scopeElem.Options, &taskOptions) + if err != nil { + return nil, err + } + taskOptions["connectionId"] = connectionId + + //TODO Add transformation rules to task options + + /* + var transformationRules tasks.TransformationRules + if len(scopeElem.Transformation) > 0 { + err = json.Unmarshal(scopeElem.Transformation, &transformationRules) + if err != nil { + return nil, err + } + } + */ + //taskOptions["transformationRules"] = transformationRules + _, err := tasks.DecodeAndValidateTaskOptions(taskOptions) + if err != nil { + return nil, err + } + // subtasks + subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, scopeElem.Entities) + if err != nil { + return nil, err + } + plan[i] = core.PipelineStage{ + { + Plugin: "zentao", + Subtasks: subtasks, + Options: taskOptions, + }, + } + } + return plan, nil +} diff --git a/plugins/zentao/api/connection.go b/plugins/zentao/api/connection.go new file mode 100644 index 00000000..2f8183d5 --- /dev/null +++ b/plugins/zentao/api/connection.go @@ -0,0 +1,149 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "github.com/apache/incubator-devlake/errors" + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/feishu/apimodels" + "github.com/apache/incubator-devlake/plugins/helper" + "github.com/apache/incubator-devlake/plugins/zentao/models" + "github.com/mitchellh/mapstructure" + "net/http" +) + +//TODO Please modify the following code to fit your needs +func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { + // process input + var params models.TestConnectionRequest + err := mapstructure.Decode(input.Body, ¶ms) + if err != nil { + return nil, errors.BadInput.Wrap(err, "could not decode request parameters", errors.AsUserMessage()) + } + err = vld.Struct(params) + if err != nil { + return nil, errors.BadInput.Wrap(err, "could not validate request parameters", errors.AsUserMessage()) + } + + authApiClient, err := helper.NewApiClient(context.TODO(), params.Endpoint, nil, 0, params.Proxy, basicRes) + if err != nil { + return nil, err + } + + // request for access token + tokenReqBody := &apimodels.ApiAccessTokenRequest{ + AppId: params.Username, + AppSecret: params.Password, + } + tokenRes, err := authApiClient.Post("/tokens", nil, tokenReqBody, nil) + if err != nil { + return nil, err + } + tokenResBody := &apimodels.ApiAccessTokenResponse{} + err = helper.UnmarshalResponse(tokenRes, tokenResBody) + if err != nil { + return nil, err + } + if tokenResBody.AppAccessToken == "" && tokenResBody.TenantAccessToken == "" { + return nil, errors.Default.New("failed to request access token") + } + + // output + return nil, nil +} + +//TODO Please modify the folowing code to adapt to your plugin +/* +POST /plugins/Zentao/connections +{ + "name": "Zentao data connection name", + "endpoint": "Zentao api endpoint, i.e. https://example.com";, + "username": "username, usually should be email address", + "password": "Zentao api access token" +} +*/ +func PostConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { + // update from request and save to database + connection := &models.ZentaoConnection{} + err := connectionHelper.Create(connection, input) + if err != nil { + return nil, err + } + return &core.ApiResourceOutput{Body: connection, Status: http.StatusOK}, nil +} + +//TODO Please modify the folowing code to adapt to your plugin +/* +PATCH /plugins/Zentao/connections/:connectionId +{ + "name": "Zentao data connection name", + "endpoint": "Zentao api endpoint, i.e. https://example.com";, + "username": "username, usually should be email address", + "password": "Zentao api access token" +} +*/ +func PatchConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { + connection := &models.ZentaoConnection{} + err := connectionHelper.Patch(connection, input) + if err != nil { + return nil, err + } + return &core.ApiResourceOutput{Body: connection}, nil +} + +/* +DELETE /plugins/Zentao/connections/:connectionId +*/ +func DeleteConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { + connection := &models.ZentaoConnection{} + err := connectionHelper.First(connection, input.Params) + if err != nil { + return nil, err + } + err = connectionHelper.Delete(connection) + return &core.ApiResourceOutput{Body: connection}, err +} + +/* +GET /plugins/Zentao/connections +*/ +func ListConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { + var connections []models.ZentaoConnection + err := connectionHelper.List(&connections) + if err != nil { + return nil, err + } + return &core.ApiResourceOutput{Body: connections, Status: http.StatusOK}, nil +} + +//TODO Please modify the folowing code to adapt to your plugin +/* +GET /plugins/Zentao/connections/:connectionId +{ + "name": "Zentao data connection name", + "endpoint": "Zentao api endpoint, i.e. https://merico.atlassian.net/rest";, + "username": "username, usually should be email address", + "password": "Zentao api access token" +} +*/ +func GetConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, error) { + connection := &models.ZentaoConnection{} + err := connectionHelper.First(connection, input.Params) + return &core.ApiResourceOutput{Body: connection}, err +} diff --git a/plugins/zentao/api/init.go b/plugins/zentao/api/init.go new file mode 100644 index 00000000..6774e148 --- /dev/null +++ b/plugins/zentao/api/init.go @@ -0,0 +1,39 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/helper" + "github.com/go-playground/validator/v10" + "github.com/spf13/viper" + "gorm.io/gorm" +) + +var vld *validator.Validate +var connectionHelper *helper.ConnectionApiHelper +var basicRes core.BasicRes + +func Init(config *viper.Viper, logger core.Logger, database *gorm.DB) { + basicRes = helper.NewDefaultBasicRes(config, logger, database) + vld = validator.New() + connectionHelper = helper.NewConnectionHelper( + basicRes, + vld, + ) +} diff --git a/plugins/zentao/impl/impl.go b/plugins/zentao/impl/impl.go new file mode 100644 index 00000000..cc81c226 --- /dev/null +++ b/plugins/zentao/impl/impl.go @@ -0,0 +1,124 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "fmt" + "github.com/apache/incubator-devlake/migration" + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/zentao/api" + "github.com/apache/incubator-devlake/plugins/zentao/models" + "github.com/apache/incubator-devlake/plugins/zentao/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/zentao/tasks" + "github.com/apache/incubator-devlake/plugins/helper" + "github.com/spf13/viper" + "gorm.io/gorm" +) + +// make sure interface is implemented +var _ core.PluginMeta = (*Zentao)(nil) +var _ core.PluginInit = (*Zentao)(nil) +var _ core.PluginTask = (*Zentao)(nil) +var _ core.PluginApi = (*Zentao)(nil) +var _ core.PluginBlueprintV100 = (*Zentao)(nil) +var _ core.CloseablePluginTask = (*Zentao)(nil) + + + +type Zentao struct{} + +func (plugin Zentao) Description() string { + return "collect some Zentao data" +} + +func (plugin Zentao) Init(config *viper.Viper, logger core.Logger, db *gorm.DB) error { + api.Init(config, logger, db) + return nil +} + +func (plugin Zentao) SubTaskMetas() []core.SubTaskMeta { + // TODO add your sub task here + return []core.SubTaskMeta{ + tasks.CollectProjectMeta, + } +} + +func (plugin Zentao) PrepareTaskData(taskCtx core.TaskContext, options map[string]interface{}) (interface{}, error) { + op, err := tasks.DecodeAndValidateTaskOptions(options) + if err != nil { + return nil, err + } + connectionHelper := helper.NewConnectionHelper( + taskCtx, + nil, + ) + connection := &models.ZentaoConnection{} + err = connectionHelper.FirstById(connection, op.ConnectionId) + if err != nil { + return nil, fmt.Errorf("unable to get Zentao connection by the given connection ID: %v", err) + } + + apiClient, err := tasks.NewZentaoApiClient(taskCtx, connection) + if err != nil { + return nil, fmt.Errorf("unable to get Zentao API client instance: %v", err) + } + + return &tasks.ZentaoTaskData{ + Options: op, + ApiClient: apiClient, + }, nil +} + +// PkgPath information lost when compiled as plugin(.so) +func (plugin Zentao) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/zentao" +} + +func (plugin Zentao) MigrationScripts() []migration.Script { + return migrationscripts.All() +} + +func (plugin Zentao) ApiResources() map[string]map[string]core.ApiResourceHandler { + return map[string]map[string]core.ApiResourceHandler{ + "test": { + "POST": api.TestConnection, + }, + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "GET": api.GetConnection, + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + }, + } +} + +func (plugin Zentao) MakePipelinePlan(connectionId uint64, scope []*core.BlueprintScopeV100) (core.PipelinePlan, error) { + return api.MakePipelinePlan(plugin.SubTaskMetas(), connectionId, scope) +} + +func (plugin Zentao) Close(taskCtx core.TaskContext) error { + data, ok := taskCtx.GetData().(*tasks.ZentaoTaskData) + if !ok { + return fmt.Errorf("GetData failed when try to close %+v", taskCtx) + } + data.ApiClient.Release() + return nil +} diff --git a/plugins/zentao/models/archived/connection.go b/plugins/zentao/models/archived/connection.go new file mode 100644 index 00000000..60d7178c --- /dev/null +++ b/plugins/zentao/models/archived/connection.go @@ -0,0 +1,70 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/models/migrationscripts/archived" +) + +//TODO Please modify the following code to fit your needs +// This object conforms to what the frontend currently sends. +type ZentaoConnection struct { + RestConnection `mapstructure:",squash"` + //TODO you may need to use helper.BasicAuth instead of helper.AccessToken + BasicAuth `mapstructure:",squash"` +} + +type TestConnectionRequest struct { + Endpoint string `json:"endpoint"` + Proxy string `json:"proxy"` + BasicAuth `mapstructure:",squash"` +} + +// This object conforms to what the frontend currently expects. +type ZentaoResponse struct { + Name string `json:"name"` + ID int `json:"id"` + ZentaoConnection +} + +// Using User because it requires authentication. +type ApiUserResponse struct { + Id int + Name string `json:"name"` +} + +func (ZentaoConnection) TableName() string { + return "_tool_zentao_connections" +} + +type BasicAuth struct { + Username string `mapstructure:"username" validate:"required" json:"username"` + Password string `mapstructure:"password" validate:"required" json:"password"` +} + +type RestConnection struct { + BaseConnection `mapstructure:",squash"` + Endpoint string `mapstructure:"endpoint" validate:"required" json:"endpoint"` + Proxy string `mapstructure:"proxy" json:"proxy"` + RateLimitPerHour int `comment:"api request rate limt per hour" json:"rateLimit"` +} + +type BaseConnection struct { + Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"` + archived.Model +} diff --git a/plugins/zentao/models/connection.go b/plugins/zentao/models/connection.go new file mode 100644 index 00000000..6140d7f4 --- /dev/null +++ b/plugins/zentao/models/connection.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import "github.com/apache/incubator-devlake/plugins/helper" + +//TODO Please modify the following code to fit your needs +// This object conforms to what the frontend currently sends. +type ZentaoConnection struct { + helper.RestConnection `mapstructure:",squash"` + //TODO you may need to use helper.BasicAuth instead of helper.AccessToken + helper.BasicAuth `mapstructure:",squash"` +} + +type TestConnectionRequest struct { + Endpoint string `json:"endpoint"` + Proxy string `json:"proxy"` + helper.BasicAuth `mapstructure:",squash"` +} + +// This object conforms to what the frontend currently expects. +type ZentaoResponse struct { + Name string `json:"name"` + ID int `json:"id"` + ZentaoConnection +} + +// Using User because it requires authentication. +type ApiUserResponse struct { + Id int + Name string `json:"name"` +} + +func (ZentaoConnection) TableName() string { + return "_tool_zentao_connections" +} diff --git a/plugins/zentao/models/migrationscripts/20220906_add_init_tables.go b/plugins/zentao/models/migrationscripts/20220906_add_init_tables.go new file mode 100644 index 00000000..181bbc77 --- /dev/null +++ b/plugins/zentao/models/migrationscripts/20220906_add_init_tables.go @@ -0,0 +1,39 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "context" + "gorm.io/gorm" +) + +type addInitTables struct {} + +func (u *addInitTables) Up(ctx context.Context, db *gorm.DB) error { + return db.Migrator().AutoMigrate( + // TODO add you models + ) +} + +func (*addInitTables) Version() uint64 { + return 20220906000001 +} + +func (*addInitTables) Name() string { + return "zentao init schemas" +} diff --git a/plugins/zentao/models/migrationscripts/register.go b/plugins/zentao/models/migrationscripts/register.go new file mode 100644 index 00000000..92e20c01 --- /dev/null +++ b/plugins/zentao/models/migrationscripts/register.go @@ -0,0 +1,27 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import "github.com/apache/incubator-devlake/migration" + +// All return all the migration scripts +func All() []migration.Script { + return []migration.Script{ + new(addInitTables), + } +} diff --git a/plugins/zentao/tasks/api_client.go b/plugins/zentao/tasks/api_client.go new file mode 100644 index 00000000..62d2d954 --- /dev/null +++ b/plugins/zentao/tasks/api_client.go @@ -0,0 +1,92 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "github.com/apache/incubator-devlake/errors" + "github.com/apache/incubator-devlake/plugins/feishu/apimodels" + "net/http" + "strconv" + "time" + + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/helper" + "github.com/apache/incubator-devlake/plugins/zentao/models" +) + +func NewZentaoApiClient(taskCtx core.TaskContext, connection *models.ZentaoConnection) (*helper.ApiAsyncClient, error) { + authApiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.Endpoint, nil, 0, connection.Proxy, taskCtx) + if err != nil { + return nil, err + } + + // request for access token + tokenReqBody := &apimodels.ApiAccessTokenRequest{ + AppId: connection.Username, + AppSecret: connection.Password, + } + tokenRes, err := authApiClient.Post("/tokens", nil, tokenReqBody, nil) + if err != nil { + return nil, err + } + tokenResBody := &apimodels.ApiAccessTokenResponse{} + err = helper.UnmarshalResponse(tokenRes, tokenResBody) + if err != nil { + return nil, err + } + if tokenResBody.AppAccessToken == "" && tokenResBody.TenantAccessToken == "" { + return nil, errors.Default.New("failed to request access token") + } + // real request apiClient + apiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.Endpoint, nil, 0, connection.Proxy, taskCtx) + if err != nil { + return nil, err + } + // set token + apiClient.SetHeaders(map[string]string{ + "Token": fmt.Sprintf("%v", tokenResBody.TenantAccessToken), + }) + + // create rate limit calculator + rateLimiter := &helper.ApiRateLimitCalculator{ + UserRateLimitPerHour: connection.RateLimitPerHour, + DynamicRateLimit: func(res *http.Response) (int, time.Duration, error) { + rateLimitHeader := res.Header.Get("RateLimit-Limit") + if rateLimitHeader == "" { + // use default + return 0, 0, nil + } + rateLimit, err := strconv.Atoi(rateLimitHeader) + if err != nil { + return 0, 0, fmt.Errorf("failed to parse RateLimit-Limit header: %w", err) + } + // seems like {{ .plugin-ame }} rate limit is on minute basis + return rateLimit, 1 * time.Minute, nil + }, + } + asyncApiClient, err := helper.CreateAsyncApiClient( + taskCtx, + apiClient, + rateLimiter, + ) + if err != nil { + return nil, err + } + return asyncApiClient, nil +} diff --git a/plugins/zentao/tasks/project_collector.go b/plugins/zentao/tasks/project_collector.go new file mode 100644 index 00000000..7af9dc2c --- /dev/null +++ b/plugins/zentao/tasks/project_collector.go @@ -0,0 +1,78 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/helper" + "net/http" + "net/url" +) + +const RAW_PROJECT_TABLE = "zentao_project" + +var _ core.SubTaskEntryPoint = CollectProject + +func CollectProject(taskCtx core.SubTaskContext) error { + data := taskCtx.GetData().(*ZentaoTaskData) + iterator, err := helper.NewDateIterator(365) + if err != nil { + return err + } + + collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: ZentaoApiParams{}, + Table: RAW_PROJECT_TABLE, + }, + ApiClient: data.ApiClient, + Incremental: false, + Input: iterator, + PageSize: 100, + // TODO write which api would you want request + UrlTemplate: "projects", + Query: func(reqData *helper.RequestData) (url.Values, error) { + query := url.Values{} + query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page)) + query.Set("limit", fmt.Sprintf("%v", reqData.Pager.Size)) + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, error) { + var data struct { + Projects []json.RawMessage `json:"data"` + } + err = helper.UnmarshalResponse(res, &data) + return data.Projects, err + }, + }) + if err != nil { + return err + } + + return collector.Execute() +} + +var CollectProjectMeta = core.SubTaskMeta{ + Name: "CollectProject", + EntryPoint: CollectProject, + EnabledByDefault: true, + Description: "Collect Project data from Zentao api", +} diff --git a/plugins/zentao/tasks/task_data.go b/plugins/zentao/tasks/task_data.go new file mode 100644 index 00000000..a2d88f22 --- /dev/null +++ b/plugins/zentao/tasks/task_data.go @@ -0,0 +1,61 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "github.com/apache/incubator-devlake/plugins/helper" + "github.com/mitchellh/mapstructure" +) + +type ZentaoApiParams struct { + ProductId uint64 + ExecutionId uint64 + ProjectId uint64 +} + +type ZentaoOptions struct { + // TODO add some custom options here if necessary + // options means some custom params required by plugin running. + // Such As How many rows do your want + // You can use it in sub tasks and you need pass it in main.go and pipelines. + ConnectionId uint64 `json:"connectionId"` + ProductId uint64 + ExecutionId uint64 + ProjectId uint64 + Tasks []string `json:"tasks,omitempty"` + Since string +} + +type ZentaoTaskData struct { + Options *ZentaoOptions + ApiClient *helper.ApiAsyncClient +} + +func DecodeAndValidateTaskOptions(options map[string]interface{}) (*ZentaoOptions, error) { + var op ZentaoOptions + err := mapstructure.Decode(options, &op) + if err != nil { + return nil, err + } + + if op.ConnectionId == 0 { + return nil, fmt.Errorf("connectionId is invalid") + } + return &op, nil +} diff --git a/plugins/zentao/zentao.go b/plugins/zentao/zentao.go new file mode 100644 index 00000000..32845c90 --- /dev/null +++ b/plugins/zentao/zentao.go @@ -0,0 +1,47 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/apache/incubator-devlake/plugins/Zentao/impl" + "github.com/apache/incubator-devlake/runner" + "github.com/spf13/cobra" +) + +// Export a variable named PluginEntry for Framework to search and load +var PluginEntry impl.Zentao //nolint + +// standalone mode for debugging +func main() { + cmd := &cobra.Command{Use: "zentao"} + + connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "zentao connection id") + executionId := cmd.Flags().IntP("executionId", "e", 8, "execution id") + productId := cmd.Flags().IntP("productId", "o", 8, "product id") + projectId := cmd.Flags().IntP("projectId", "p", 8, "project id") + + cmd.Run = func(cmd *cobra.Command, args []string) { + runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{ + "connectionId": *connectionId, + "executionId": *executionId, + "productId": *productId, + "projectId": *projectId, + }) + } + runner.RunCmd(cmd) +}
