This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 107a5d33 fix(framework): update code generator templates
107a5d33 is described below
commit 107a5d33c3f3dadda4c929efcbdc43cbaf311925
Author: Yingchu Chen <[email protected]>
AuthorDate: Sat Jul 30 15:32:18 2022 +0800
fix(framework): update code generator templates
closes #2640
---
generator/cmd/create_collector.go | 5 +
generator/cmd/create_extractor.go | 5 +
generator/cmd/create_plugin.go | 35 ++---
.../archived/connection.go-template | 0
.../template/plugin/api/blueprint.go-template | 69 +++++++++
.../template/plugin/api/connection.go-template | 155 +++++++++++++++++++++
.../init.go-template} | 28 ++--
.../plugin/impl/impl_complete_plugin.go-template | 123 ++++++++++++++++
.../template/plugin/models/connection.go-template | 51 +++++++
...ate => plugin_main_complete_plugin.go-template} | 32 +++--
.../plugin/plugin_main_with_api_client.go-template | 100 -------------
.../template/plugin/tasks/api_client.go-template | 71 +++++-----
...plate => task_data_complete_plugin.go-template} | 18 +++
13 files changed, 510 insertions(+), 182 deletions(-)
diff --git a/generator/cmd/create_collector.go
b/generator/cmd/create_collector.go
index 6c5f3d78..bef7385f 100644
--- a/generator/cmd/create_collector.go
+++ b/generator/cmd/create_collector.go
@@ -134,6 +134,11 @@ Type in what the name of collector is, then generator will
create a new collecto
util.ReplaceVarInTemplates(templates, values)
util.WriteTemplates(filepath.Join(`plugins`, pluginName,
`tasks`), templates)
if modifyExistCode {
+ util.ReplaceVarInFile(
+ filepath.Join(`plugins`, pluginName,
`impl/impl.go`),
+ regexp.MustCompile(`(return
+\[]core\.SubTaskMeta ?\{ ?\n?)((\s*[\w.]+,\n)*)(\s*})`),
+ fmt.Sprintf("$1$2\t\ttasks.Collect%sMeta,\n$4",
collectorDataNameUpperCamel),
+ )
util.ReplaceVarInFile(
filepath.Join(`plugins`, pluginName,
`plugin_main.go`),
regexp.MustCompile(`(return
+\[]core\.SubTaskMeta ?\{ ?\n?)((\s*[\w.]+,\n)*)(\s*})`),
diff --git a/generator/cmd/create_extractor.go
b/generator/cmd/create_extractor.go
index b63ac6a6..d16c3d7f 100644
--- a/generator/cmd/create_extractor.go
+++ b/generator/cmd/create_extractor.go
@@ -121,6 +121,11 @@ Type in what the name of extractor is, then generator will
create a new extracto
regexp.MustCompile(`(return
+\[]core\.SubTaskMeta ?\{ ?\n?)((\s*[\w.]+,\n)*)(\s*})`),
fmt.Sprintf("$1$2\t\ttasks.Extract%sMeta,\n$4",
extractorDataNameUpperCamel),
)
+ util.ReplaceVarInFile(
+ filepath.Join(`plugins`, pluginName,
`impl/impl.go`),
+ regexp.MustCompile(`(return
+\[]core\.SubTaskMeta ?\{ ?\n?)((\s*[\w.]+,\n)*)(\s*})`),
+ fmt.Sprintf("$1$2\t\ttasks.Extract%sMeta,\n$4",
extractorDataNameUpperCamel),
+ )
}
},
}
diff --git a/generator/cmd/create_plugin.go b/generator/cmd/create_plugin.go
index 59a1ad2f..23b9d02b 100644
--- a/generator/cmd/create_plugin.go
+++ b/generator/cmd/create_plugin.go
@@ -27,6 +27,7 @@ import (
"os"
"regexp"
"strings"
+ "time"
)
func init() {
@@ -103,7 +104,7 @@ Type in what the name of plugin is, then generator will
create a new plugin in p
}
prompt := promptui.Select{
- Label: "with_api_client (Will this plugin request HTTP
APIs?)",
+ Label: "complete_plugin (Will this plugin request HTTP
APIs?)",
Items: []string{"Yes", "No"},
}
_, withApiClient, err := prompt.Run()
@@ -112,30 +113,22 @@ Type in what the name of plugin is, then generator will
create a new plugin in p
values := map[string]string{}
templates := map[string]string{}
if withApiClient == `Yes` {
- prompt := promptui.Prompt{
- Label: "Endpoint (which host to request)",
- Default: `https://open.example.cn/api/v1`,
- Validate: func(input string) error {
- if input == `` {
- return errors.New("endpoint
require")
- }
- if !strings.HasPrefix(input, `http`) {
- return errors.New("endpoint
should start with http")
- }
- return nil
- },
- }
- endpoint, err := prompt.Run()
- cobra.CheckErr(err)
-
+ versionTimestamp := time.Now().Format(`20060102`)
+ values[`Date`] = versionTimestamp
// read template
templates = map[string]string{
- `plugin_main.go`:
util.ReadTemplate("generator/template/plugin/plugin_main_with_api_client.go-template"),
- `tasks/api_client.go`:
util.ReadTemplate("generator/template/plugin/tasks/api_client.go-template"),
- `tasks/task_data.go`:
util.ReadTemplate("generator/template/plugin/tasks/task_data_with_api_client.go-template"),
+ fmt.Sprintf(`%s.go`, pluginName):
util.ReadTemplate("generator/template/plugin/plugin_main_complete_plugin.go-template"),
+ `impl/impl.go`:
util.ReadTemplate("generator/template/plugin/impl/impl_complete_plugin.go-template"),
+ `tasks/api_client.go`:
util.ReadTemplate("generator/template/plugin/tasks/api_client.go-template"),
+ `tasks/task_data.go`:
util.ReadTemplate("generator/template/plugin/tasks/task_data_complete_plugin.go-template"),
+ `api/connection.go`:
util.ReadTemplate("generator/template/plugin/api/connection.go-template"),
+ `models/connection.go`:
util.ReadTemplate("generator/template/plugin/models/connection.go-template"),
+
fmt.Sprintf("models/migrationscripts/%s_add_init_tables.go", versionTimestamp):
util.ReadTemplate("generator/template/migrationscripts/add_init_tables.go-template"),
+ `models/migrationscripts/register.go`:
util.ReadTemplate("generator/template/migrationscripts/register.go-template"),
+ `api/init.go`:
util.ReadTemplate("generator/template/plugin/api/init.go-template"),
+ `api/blueprint.go`:
util.ReadTemplate("generator/template/plugin/api/blueprint.go-template"),
}
util.GenerateAllFormatVar(values, `plugin_name`,
pluginName)
- values[`Endpoint`] = endpoint
} else if withApiClient == `No` {
// read template
templates = map[string]string{
diff --git
a/generator/template/migrationscripts/archived/connection.go-template
b/generator/template/migrationscripts/archived/connection.go-template
new file mode 100644
index 00000000..e69de29b
diff --git a/generator/template/plugin/api/blueprint.go-template
b/generator/template/plugin/api/blueprint.go-template
new file mode 100644
index 00000000..cdf40067
--- /dev/null
+++ b/generator/template/plugin/api/blueprint.go-template
@@ -0,0 +1,69 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "encoding/json"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/plugins/{{ .plugin_name }}/tasks"
+)
+
+func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64,
scope []*core.BlueprintScopeV100) (core.PipelinePlan, error) {
+ var err error
+ plan := make(core.PipelinePlan, len(scope))
+ for i, scopeElem := range scope {
+ taskOptions := make(map[string]interface{})
+ err = json.Unmarshal(scopeElem.Options, &taskOptions)
+ if err != nil {
+ return nil, err
+ }
+ taskOptions["connectionId"] = connectionId
+
+ //TODO Add transformation rules to task options
+
+ /*
+ var transformationRules tasks.TransformationRules
+ if len(scopeElem.Transformation) > 0 {
+ err = json.Unmarshal(scopeElem.Transformation,
&transformationRules)
+ if err != nil {
+ return nil, err
+ }
+ }
+ */
+ //taskOptions["transformationRules"] = transformationRules
+ _, err := tasks.DecodeAndValidateTaskOptions(taskOptions)
+ if err != nil {
+ return nil, err
+ }
+ // subtasks
+ subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas,
scopeElem.Entities)
+ if err != nil {
+ return nil, err
+ }
+ plan[i] = core.PipelineStage{
+ {
+ Plugin: "{{ .plugin_name }}",
+ Subtasks: subtasks,
+ Options: taskOptions,
+ },
+ }
+ }
+ return plan, nil
+}
diff --git a/generator/template/plugin/api/connection.go-template
b/generator/template/plugin/api/connection.go-template
new file mode 100644
index 00000000..0e781d27
--- /dev/null
+++ b/generator/template/plugin/api/connection.go-template
@@ -0,0 +1,155 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "time"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/plugins/{{ .plugin_name }}/models"
+ "github.com/mitchellh/mapstructure"
+)
+
+//TODO Please modify the following code to fit your needs
+func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput,
error) {
+ // decode
+ var err error
+ var connection models.TestConnectionRequest
+ err = mapstructure.Decode(input.Body, &connection)
+ if err != nil {
+ return nil, err
+ }
+ // validate
+ err = vld.Struct(connection)
+ if err != nil {
+ return nil, err
+ }
+ // test connection
+ apiClient, err := helper.NewApiClient(
+ context.TODO(),
+ connection.Endpoint,
+ map[string]string{
+ "Authorization": fmt.Sprintf("Bearer %v",
connection.Token),
+ },
+ 3*time.Second,
+ connection.Proxy,
+ basicRes,
+ )
+ if err != nil {
+ return nil, err
+ }
+
+ res, err := apiClient.Get("user", nil, nil)
+ if err != nil {
+ return nil, err
+ }
+ resBody := &models.ApiUserResponse{}
+ err = helper.UnmarshalResponse(res, resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ if res.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("unexpected status code: %d",
res.StatusCode)
+ }
+ return nil, nil
+}
+
+//TODO Please modify the folowing code to adapt to your plugin
+/*
+POST /plugins/{{ .PluginName }}/connections
+{
+ "name": "{{ .PluginName }} data connection name",
+ "endpoint": "{{ .PluginName }} api endpoint, i.e. https://example.com";,
+ "username": "username, usually should be email address",
+ "password": "{{ .PluginName }} api access token"
+}
+*/
+func PostConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput,
error) {
+ // update from request and save to database
+ connection := &models.{{ .PluginName }}Connection{}
+ err := connectionHelper.Create(connection, input)
+ if err != nil {
+ return nil, err
+ }
+ return &core.ApiResourceOutput{Body: connection, Status:
http.StatusOK}, nil
+}
+
+//TODO Please modify the folowing code to adapt to your plugin
+/*
+PATCH /plugins/{{ .PluginName }}/connections/:connectionId
+{
+ "name": "{{ .PluginName }} data connection name",
+ "endpoint": "{{ .PluginName }} api endpoint, i.e. https://example.com";,
+ "username": "username, usually should be email address",
+ "password": "{{ .PluginName }} api access token"
+}
+*/
+func PatchConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput,
error) {
+ connection := &models.{{ .PluginName }}Connection{}
+ err := connectionHelper.Patch(connection, input)
+ if err != nil {
+ return nil, err
+ }
+ return &core.ApiResourceOutput{Body: connection}, nil
+}
+
+/*
+DELETE /plugins/{{ .PluginName }}/connections/:connectionId
+*/
+func DeleteConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput,
error) {
+ connection := &models.{{ .PluginName }}Connection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+ err = connectionHelper.Delete(connection)
+ return &core.ApiResourceOutput{Body: connection}, err
+}
+
+/*
+GET /plugins/{{ .PluginName }}/connections
+*/
+func ListConnections(input *core.ApiResourceInput) (*core.ApiResourceOutput,
error) {
+ var connections []models.{{ .PluginName }}Connection
+ err := connectionHelper.List(&connections)
+ if err != nil {
+ return nil, err
+ }
+ return &core.ApiResourceOutput{Body: connections, Status:
http.StatusOK}, nil
+}
+
+//TODO Please modify the folowing code to adapt to your plugin
+/*
+GET /plugins/{{ .PluginName }}/connections/:connectionId
+{
+ "name": "{{ .PluginName }} data connection name",
+ "endpoint": "{{ .PluginName }} api endpoint, i.e.
https://merico.atlassian.net/rest";,
+ "username": "username, usually should be email address",
+ "password": "{{ .PluginName }} api access token"
+}
+*/
+func GetConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput,
error) {
+ connection := &models.{{ .PluginName }}Connection{}
+ err := connectionHelper.First(connection, input.Params)
+ return &core.ApiResourceOutput{Body: connection}, err
+}
\ No newline at end of file
diff --git
a/generator/template/plugin/tasks/task_data_with_api_client.go-template
b/generator/template/plugin/api/init.go-template
similarity index 63%
copy from generator/template/plugin/tasks/task_data_with_api_client.go-template
copy to generator/template/plugin/api/init.go-template
index 5dec7665..6774e148 100644
--- a/generator/template/plugin/tasks/task_data_with_api_client.go-template
+++ b/generator/template/plugin/api/init.go-template
@@ -15,23 +15,25 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package tasks
+package api
import (
+ "github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/go-playground/validator/v10"
+ "github.com/spf13/viper"
+ "gorm.io/gorm"
)
-type {{ .PluginName }}ApiParams struct {
-}
-
-type {{ .PluginName }}Options struct {
- // TODO add some custom options here if necessary
- // options means some custom params required by plugin running.
- // Such As How many rows do your want
- // You can use it in sub tasks and you need pass it in main.go and
pipelines.
-}
+var vld *validator.Validate
+var connectionHelper *helper.ConnectionApiHelper
+var basicRes core.BasicRes
-type {{ .PluginName }}TaskData struct {
- Options *{{ .PluginName }}Options
- ApiClient *helper.ApiAsyncClient
+func Init(config *viper.Viper, logger core.Logger, database *gorm.DB) {
+ basicRes = helper.NewDefaultBasicRes(config, logger, database)
+ vld = validator.New()
+ connectionHelper = helper.NewConnectionHelper(
+ basicRes,
+ vld,
+ )
}
diff --git a/generator/template/plugin/impl/impl_complete_plugin.go-template
b/generator/template/plugin/impl/impl_complete_plugin.go-template
new file mode 100644
index 00000000..8258b138
--- /dev/null
+++ b/generator/template/plugin/impl/impl_complete_plugin.go-template
@@ -0,0 +1,123 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package impl
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/migration"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/{{ .plugin_name }}/api"
+ "github.com/apache/incubator-devlake/plugins/{{ .plugin_name }}/models"
+ "github.com/apache/incubator-devlake/plugins/{{ .plugin_name
}}/models/migrationscripts"
+ "github.com/apache/incubator-devlake/plugins/{{ .plugin_name }}/tasks"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/spf13/viper"
+ "gorm.io/gorm"
+)
+
+// make sure interface is implemented
+var _ core.PluginMeta = (*{{ .PluginName }})(nil)
+var _ core.PluginInit = (*{{ .PluginName }})(nil)
+var _ core.PluginTask = (*{{ .PluginName }})(nil)
+var _ core.PluginApi = (*{{ .PluginName }})(nil)
+var _ core.PluginBlueprintV100 = (*{{ .PluginName }})(nil)
+var _ core.CloseablePluginTask = (*{{ .PluginName }})(nil)
+
+
+
+type {{ .PluginName }} struct{}
+
+func (plugin {{ .PluginName }}) Description() string {
+ return "collect some {{ .PluginName }} data"
+}
+
+func (plugin {{ .PluginName }}) Init(config *viper.Viper, logger core.Logger,
db *gorm.DB) error {
+ api.Init(config, logger, db)
+ return nil
+}
+
+func (plugin {{ .PluginName }}) SubTaskMetas() []core.SubTaskMeta {
+ // TODO add your sub task here
+ return []core.SubTaskMeta{
+ }
+}
+
+func (plugin {{ .PluginName }}) PrepareTaskData(taskCtx core.TaskContext,
options map[string]interface{}) (interface{}, error) {
+ op, err := tasks.DecodeAndValidateTaskOptions(options)
+ if err != nil {
+ return nil, err
+ }
+ connectionHelper := helper.NewConnectionHelper(
+ taskCtx,
+ nil,
+ )
+ connection := &models.{{ .PluginName }}Connection{}
+ err = connectionHelper.FirstById(connection, op.ConnectionId)
+ if err != nil {
+ return nil, fmt.Errorf("unable to get {{ .PluginName }} connection by
the given connection ID: %v", err)
+ }
+
+ apiClient, err := tasks.New{{ .PluginName }}ApiClient(taskCtx, connection)
+ if err != nil {
+ return nil, fmt.Errorf("unable to get {{ .PluginName }} API client
instance: %v", err)
+ }
+
+ return &tasks.{{ .PluginName }}TaskData{
+ Options: op,
+ ApiClient: apiClient,
+ }, nil
+}
+
+// PkgPath information lost when compiled as plugin(.so)
+func (plugin {{ .PluginName }}) RootPkgPath() string {
+ return "github.com/apache/incubator-devlake/plugins/{{ .plugin_name }}"
+}
+
+func (plugin {{ .PluginName }}) MigrationScripts() []migration.Script {
+ return migrationscripts.All()
+}
+
+func (plugin {{ .PluginName }}) ApiResources()
map[string]map[string]core.ApiResourceHandler {
+ return map[string]map[string]core.ApiResourceHandler{
+ "test": {
+ "POST": api.TestConnection,
+ },
+ "connections": {
+ "POST": api.PostConnections,
+ "GET": api.ListConnections,
+ },
+ "connections/:connectionId": {
+ "GET": api.GetConnection,
+ "PATCH": api.PatchConnection,
+ "DELETE": api.DeleteConnection,
+ },
+ }
+}
+
+func (plugin {{ .PluginName }}) MakePipelinePlan(connectionId uint64, scope
[]*core.BlueprintScopeV100) (core.PipelinePlan, error) {
+ return api.MakePipelinePlan(plugin.SubTaskMetas(), connectionId, scope)
+}
+
+func (plugin {{ .PluginName }}) Close(taskCtx core.TaskContext) error {
+ data, ok := taskCtx.GetData().(*tasks.{{ .PluginName }}TaskData)
+ if !ok {
+ return fmt.Errorf("GetData failed when try to close %+v",
taskCtx)
+ }
+ data.ApiClient.Release()
+ return nil
+}
diff --git a/generator/template/plugin/models/connection.go-template
b/generator/template/plugin/models/connection.go-template
new file mode 100644
index 00000000..72133f10
--- /dev/null
+++ b/generator/template/plugin/models/connection.go-template
@@ -0,0 +1,51 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import "github.com/apache/incubator-devlake/plugins/helper"
+
+//TODO Please modify the following code to fit your needs
+// This object conforms to what the frontend currently sends.
+type {{ .PluginName }}Connection struct {
+ helper.RestConnection `mapstructure:",squash"`
+ //TODO you may need to use helper.BasicAuth instead of helper.AccessToken
+ helper.AccessToken `mapstructure:",squash"`
+}
+
+type TestConnectionRequest struct {
+ Endpoint string `json:"endpoint"`
+ Proxy string `json:"proxy"`
+ helper.AccessToken `mapstructure:",squash"`
+}
+
+// This object conforms to what the frontend currently expects.
+type {{ .PluginName }}Response struct {
+ Name string `json:"name"`
+ ID int `json:"id"`
+ {{ .PluginName }}Connection
+}
+
+// Using User because it requires authentication.
+type ApiUserResponse struct {
+ Id int
+ Name string `json:"name"`
+}
+
+func ({{ .PluginName }}Connection) TableName() string {
+ return "_tool_{{ .plugin_name }}_connections"
+}
diff --git
a/generator/template/plugin/tasks/task_data_with_api_client.go-template
b/generator/template/plugin/plugin_main_complete_plugin.go-template
similarity index 51%
copy from generator/template/plugin/tasks/task_data_with_api_client.go-template
copy to generator/template/plugin/plugin_main_complete_plugin.go-template
index 5dec7665..ee2d4a56 100644
--- a/generator/template/plugin/tasks/task_data_with_api_client.go-template
+++ b/generator/template/plugin/plugin_main_complete_plugin.go-template
@@ -15,23 +15,29 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package tasks
+package main
import (
- "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/plugins/{{ .PluginName }}/impl"
+ "github.com/apache/incubator-devlake/runner"
+ "github.com/spf13/cobra"
)
-type {{ .PluginName }}ApiParams struct {
-}
+// Export a variable named PluginEntry for Framework to search and load
+var PluginEntry impl.{{ .PluginName }} //nolint
-type {{ .PluginName }}Options struct {
- // TODO add some custom options here if necessary
- // options means some custom params required by plugin running.
- // Such As How many rows do your want
- // You can use it in sub tasks and you need pass it in main.go and
pipelines.
-}
+// standalone mode for debugging
+func main() {
+ cmd := &cobra.Command{Use: "{{ .pluginName }}"}
+
+ // TODO add your cmd flag if necessary
+ // yourFlag := cmd.Flags().IntP("yourFlag", "y", 8, "TODO add
description here")
+ // _ = cmd.MarkFlagRequired("yourFlag")
-type {{ .PluginName }}TaskData struct {
- Options *{{ .PluginName }}Options
- ApiClient *helper.ApiAsyncClient
+ cmd.Run = func(cmd *cobra.Command, args []string) {
+ runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
+ // TODO add more custom params here
+ })
+ }
+ runner.RunCmd(cmd)
}
diff --git a/generator/template/plugin/plugin_main_with_api_client.go-template
b/generator/template/plugin/plugin_main_with_api_client.go-template
deleted file mode 100644
index 1ab8333d..00000000
--- a/generator/template/plugin/plugin_main_with_api_client.go-template
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
- "github.com/apache/incubator-devlake/plugins/core"
- "github.com/apache/incubator-devlake/plugins/{{ .pluginName }}/tasks"
- "github.com/apache/incubator-devlake/runner"
- "github.com/mitchellh/mapstructure"
- "github.com/spf13/cobra"
- "github.com/spf13/viper"
- "gorm.io/gorm"
-)
-
-// make sure interface is implemented
-var _ core.PluginMeta = (*{{ .PluginName }})(nil)
-var _ core.PluginInit = (*{{ .PluginName }})(nil)
-var _ core.PluginTask = (*{{ .PluginName }})(nil)
-var _ core.PluginApi = (*{{ .PluginName }})(nil)
-
-// Export a variable named PluginEntry for Framework to search and load
-var PluginEntry {{ .PluginName }} //nolint
-
-type {{ .PluginName }} struct{}
-
-func (plugin {{ .PluginName }}) Description() string {
- return "collect some {{ .PluginName }} data"
-}
-
-func (plugin {{ .PluginName }}) Init(config *viper.Viper, logger core.Logger,
db *gorm.DB) error {
- // AutoSchemas is a **develop** script to auto migrate models easily.
- // FIXME Don't submit it as a open source plugin
- return db.Migrator().AutoMigrate(
- // TODO add your models in here
- )
-}
-
-func (plugin {{ .PluginName }}) SubTaskMetas() []core.SubTaskMeta {
- // TODO add your sub task here
- return []core.SubTaskMeta{
- }
-}
-
-func (plugin {{ .PluginName }}) PrepareTaskData(taskCtx core.TaskContext,
options map[string]interface{}) (interface{}, error) {
- var op tasks.{{ .PluginName }}Options
- err := mapstructure.Decode(options, &op)
- if err != nil {
- return nil, err
- }
-
- apiClient, err := tasks.New{{ .PluginName }}ApiClient(taskCtx)
- if err != nil {
- return nil, err
- }
-
- return &tasks.{{ .PluginName }}TaskData{
- Options: &op,
- ApiClient: apiClient,
- }, nil
-}
-
-// PkgPath information lost when compiled as plugin(.so)
-func (plugin {{ .PluginName }}) RootPkgPath() string {
- return "github.com/apache/incubator-devlake/plugins/{{ .pluginName }}"
-}
-
-func (plugin {{ .PluginName }}) ApiResources()
map[string]map[string]core.ApiResourceHandler {
- return nil
-}
-
-// standalone mode for debugging
-func main() {
- cmd := &cobra.Command{Use: "{{ .pluginName }}"}
-
- // TODO add your cmd flag if necessary
- // yourFlag := cmd.Flags().IntP("yourFlag", "y", 8, "TODO add
description here")
- // _ = cmd.MarkFlagRequired("yourFlag")
-
- cmd.Run = func(cmd *cobra.Command, args []string) {
- runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
- // TODO add more custom params here
- })
- }
- runner.RunCmd(cmd)
-}
diff --git a/generator/template/plugin/tasks/api_client.go-template
b/generator/template/plugin/tasks/api_client.go-template
index a79e1312..296131fb 100644
--- a/generator/template/plugin/tasks/api_client.go-template
+++ b/generator/template/plugin/tasks/api_client.go-template
@@ -19,54 +19,55 @@ package tasks
import (
"fmt"
+ "net/http"
+ "strconv"
+ "time"
+ "github.com/apache/incubator-devlake/plugins/{{ .plugin_name }}/models"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
- "github.com/apache/incubator-devlake/utils"
)
-// TODO add what host would want to requist
-const ENDPOINT = "{{ .Endpoint }}"
-
-func New{{ .PluginName }}ApiClient(taskCtx core.TaskContext)
(*helper.ApiAsyncClient, error) {
- // load and process configuration
- token := taskCtx.GetConfig("{{ .PLUGIN_NAME }}_TOKEN")
- if token == "" {
- println("invalid {{ .PLUGIN_NAME }}_TOKEN, but ignore this
error now")
+func New{{ .PluginName }}ApiClient(taskCtx core.TaskContext, connection
*models.{{ .PluginName }}Connection) (*helper.ApiAsyncClient, error) {
+ // create synchronize api client so we can calculate api rate limit
dynamically
+ headers := map[string]string{
+ "Authorization": fmt.Sprintf("Bearer %v", connection.Token),
}
- userRateLimit, err := utils.StrToIntOr(taskCtx.GetConfig("{{
.PLUGIN_NAME }}_API_REQUESTS_PER_HOUR"), 18000)
+ apiClient, err := helper.NewApiClient(taskCtx.GetContext(),
connection.Endpoint, headers, 0, connection.Proxy, taskCtx)
if err != nil {
return nil, err
}
- proxy := taskCtx.GetConfig("{{ .PLUGIN_NAME }}_PROXY")
+ apiClient.SetAfterFunction(func(res *http.Response) error {
+ if res.StatusCode == http.StatusUnauthorized {
+ return fmt.Errorf("authentication failed, please check
your AccessToken")
+ }
+ return nil
+ })
- // real request apiClient
- apiClient, err := helper.NewApiClient(ENDPOINT, nil, 0, proxy,
taskCtx.GetContext())
- if err != nil {
- return nil, err
- }
- // set token
- if token != "" {
- apiClient.SetHeaders(map[string]string{
- "Authorization": fmt.Sprintf("Bearer %v", token),
- })
+ // create rate limit calculator
+ rateLimiter := &helper.ApiRateLimitCalculator{
+ UserRateLimitPerHour: connection.RateLimitPerHour,
+ DynamicRateLimit: func(res *http.Response) (int, time.Duration,
error) {
+ rateLimitHeader := res.Header.Get("RateLimit-Limit")
+ if rateLimitHeader == "" {
+ // use default
+ return 0, 0, nil
+ }
+ rateLimit, err := strconv.Atoi(rateLimitHeader)
+ if err != nil {
+ return 0, 0, fmt.Errorf("failed to parse
RateLimit-Limit header: %w", err)
+ }
+ // seems like {{ .plugin-ame }} rate limit is on minute
basis
+ return rateLimit, 1 * time.Minute, nil
+ },
}
-
- // TODO add some check after request if necessary
- // apiClient.SetAfterFunction(func(res *http.Response) error {
- // if res.StatusCode == http.StatusUnauthorized {
- // return fmt.Errorf("authentication failed, please check your
Bearer Auth Token")
- // }
- // return nil
- // })
-
- // create async api client
- asyncApiClient, err := helper.CreateAsyncApiClient(taskCtx, apiClient,
&helper.ApiRateLimitCalculator{
- UserRateLimitPerHour: userRateLimit,
- })
+ asyncApiClient, err := helper.CreateAsyncApiClient(
+ taskCtx,
+ apiClient,
+ rateLimiter,
+ )
if err != nil {
return nil, err
}
-
return asyncApiClient, nil
}
diff --git
a/generator/template/plugin/tasks/task_data_with_api_client.go-template
b/generator/template/plugin/tasks/task_data_complete_plugin.go-template
similarity index 70%
rename from
generator/template/plugin/tasks/task_data_with_api_client.go-template
rename to generator/template/plugin/tasks/task_data_complete_plugin.go-template
index 5dec7665..3284d249 100644
--- a/generator/template/plugin/tasks/task_data_with_api_client.go-template
+++ b/generator/template/plugin/tasks/task_data_complete_plugin.go-template
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "fmt"
+ "github.com/mitchellh/mapstructure"
"github.com/apache/incubator-devlake/plugins/helper"
)
@@ -29,9 +31,25 @@ type {{ .PluginName }}Options struct {
// options means some custom params required by plugin running.
// Such As How many rows do your want
// You can use it in sub tasks and you need pass it in main.go and
pipelines.
+ ConnectionId uint64 `json:"connectionId"`
+ Tasks []string `json:"tasks,omitempty"`
+ Since string
}
type {{ .PluginName }}TaskData struct {
Options *{{ .PluginName }}Options
ApiClient *helper.ApiAsyncClient
}
+
+func DecodeAndValidateTaskOptions(options map[string]interface{}) (*{{
.PluginName }}Options, error) {
+ var op {{ .PluginName }}Options
+ err := mapstructure.Decode(options, &op)
+ if err != nil {
+ return nil, err
+ }
+
+ if op.ConnectionId == 0 {
+ return nil, fmt.Errorf("connectionId is invalid")
+ }
+ return &op, nil
+}
\ No newline at end of file
