This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 1145cc954 Feat(sonarqube): projects (#4269)
1145cc954 is described below
commit 1145cc954b33bef7f50ccb2013e0599de0c60b6c
Author: Warren Chen <[email protected]>
AuthorDate: Mon Jan 30 16:41:39 2023 +0800
Feat(sonarqube): projects (#4269)
* feat(sonarqube): init
* feat(sonarqube): projects
* fix(sonarqube): rebase to main
* fix(sonarqube): fix lint
* fix(generator): fix for review
---
backend/generator/cmd/create_plugin.go | 1 +
.../template/plugin/api/blueprint.go-template | 2 +-
.../template/plugin/api/connection.go-template | 2 +-
.../generator/template/plugin/api/init.go-template | 2 +-
.../plugin/impl/impl_complete_plugin.go-template | 23 ++++--
.../template/plugin/models/connection.go-template | 2 +-
.../template/plugin/tasks/api_client.go-template | 2 +-
.../plugin/tasks/api_collector.go-template | 23 +++---
.../template/plugin/tasks/extractor.go-template | 9 ++-
.../template/plugin/tasks/shared.go-template} | 33 +++++----
.../template/plugin/tasks/task_data.go-template | 2 +-
.../tasks/task_data_complete_plugin.go-template | 25 +++++--
backend/plugins/icla/tasks/task_data.go | 2 +-
backend/plugins/sonarqube/api/connection.go | 19 ++++-
backend/plugins/sonarqube/impl/impl.go | 24 +++++--
backend/plugins/sonarqube/models/connection.go | 42 +++++++----
.../migrationscripts/20230111_add_init_tables.go | 3 +-
.../sonarqube_project.go} | 31 ++++-----
...111_add_init_tables.go => sonarqube_project.go} | 31 ++++-----
backend/plugins/sonarqube/sonarqube.go | 2 +-
backend/plugins/sonarqube/tasks/api_client.go | 4 +-
.../plugins/sonarqube/tasks/projects_collector.go | 81 ++++++++++++++++++++++
.../plugins/sonarqube/tasks/projects_extractor.go | 55 +++++++++++++++
backend/plugins/sonarqube/tasks/shared.go | 65 +++++++++++++++++
backend/plugins/sonarqube/tasks/task_data.go | 19 +++--
backend/plugins/zentao/tasks/task_data.go | 2 +-
26 files changed, 390 insertions(+), 116 deletions(-)
diff --git a/backend/generator/cmd/create_plugin.go
b/backend/generator/cmd/create_plugin.go
index 3a6a4fa23..22f8adedf 100644
--- a/backend/generator/cmd/create_plugin.go
+++ b/backend/generator/cmd/create_plugin.go
@@ -127,6 +127,7 @@ Type in what the name of plugin is, then generator will
create a new plugin in p
`impl/impl.go`:
util.ReadTemplate("generator/template/plugin/impl/impl_complete_plugin.go-template"),
`tasks/api_client.go`:
util.ReadTemplate("generator/template/plugin/tasks/api_client.go-template"),
`tasks/task_data.go`:
util.ReadTemplate("generator/template/plugin/tasks/task_data_complete_plugin.go-template"),
+ `tasks/shared.go`:
util.ReadTemplate("generator/template/plugin/tasks/shared.go-template"),
`api/connection.go`:
util.ReadTemplate("generator/template/plugin/api/connection.go-template"),
`models/connection.go`:
util.ReadTemplate("generator/template/plugin/models/connection.go-template"),
fmt.Sprintf("models/migrationscripts/%s_add_init_tables.go", versionTimestamp):
util.ReadTemplate("generator/template/migrationscripts/add_init_tables.go-template"),
diff --git a/backend/generator/template/plugin/api/blueprint.go-template
b/backend/generator/template/plugin/api/blueprint.go-template
index 3429d34ad..054caac4e 100644
--- a/backend/generator/template/plugin/api/blueprint.go-template
+++ b/backend/generator/template/plugin/api/blueprint.go-template
@@ -21,7 +21,7 @@ import (
"encoding/json"
"github.com/apache/incubator-devlake/core/errors"
core "github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/plugins/helper"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/{{ .plugin_name }}/tasks"
)
diff --git a/backend/generator/template/plugin/api/connection.go-template
b/backend/generator/template/plugin/api/connection.go-template
index 07c2f5b4f..8d3b1d3af 100644
--- a/backend/generator/template/plugin/api/connection.go-template
+++ b/backend/generator/template/plugin/api/connection.go-template
@@ -25,7 +25,7 @@ import (
"time"
core "github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/plugins/helper"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/{{ .plugin_name }}/models"
)
diff --git a/backend/generator/template/plugin/api/init.go-template
b/backend/generator/template/plugin/api/init.go-template
index f81181c0d..982c1d388 100644
--- a/backend/generator/template/plugin/api/init.go-template
+++ b/backend/generator/template/plugin/api/init.go-template
@@ -19,7 +19,7 @@ package api
import (
core "github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/plugins/helper"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/go-playground/validator/v10"
)
diff --git
a/backend/generator/template/plugin/impl/impl_complete_plugin.go-template
b/backend/generator/template/plugin/impl/impl_complete_plugin.go-template
index a6e81e147..defafca36 100644
--- a/backend/generator/template/plugin/impl/impl_complete_plugin.go-template
+++ b/backend/generator/template/plugin/impl/impl_complete_plugin.go-template
@@ -26,7 +26,7 @@ import (
"github.com/apache/incubator-devlake/plugins/{{ .plugin_name }}/models"
"github.com/apache/incubator-devlake/plugins/{{ .plugin_name
}}/models/migrationscripts"
"github.com/apache/incubator-devlake/plugins/{{ .plugin_name }}/tasks"
- "github.com/apache/incubator-devlake/plugins/helper"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/spf13/viper"
)
@@ -76,11 +76,22 @@ func (p {{ .PluginName }}) PrepareTaskData(taskCtx
plugin.TaskContext, options m
if err != nil {
return nil, errors.Default.Wrap(err, "unable to get {{ .PluginName }}
API client instance")
}
-
- return &tasks.{{ .PluginName }}TaskData{
- Options: op,
- ApiClient: apiClient,
- }, nil
+ taskData := &tasks.{{ .PluginName }}TaskData{
+ Options: op,
+ ApiClient: apiClient,
+ }
+ var createdDateAfter time.Time
+ if op.CreatedDateAfter != "" {
+ createdDateAfter, err =
errors.Convert01(time.Parse(time.RFC3339, op.CreatedDateAfter))
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "invalid value
for `createdDateAfter`")
+ }
+ }
+ if !createdDateAfter.IsZero() {
+ taskData.CreatedDateAfter = &createdDateAfter
+ logger.Debug("collect data updated createdDateAfter %s",
createdDateAfter)
+ }
+ return taskData, nil
}
// PkgPath information lost when compiled as plugin(.so)
diff --git a/backend/generator/template/plugin/models/connection.go-template
b/backend/generator/template/plugin/models/connection.go-template
index 72133f103..a7d7a20b7 100644
--- a/backend/generator/template/plugin/models/connection.go-template
+++ b/backend/generator/template/plugin/models/connection.go-template
@@ -17,7 +17,7 @@ limitations under the License.
package models
-import "github.com/apache/incubator-devlake/plugins/helper"
+import helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
//TODO Please modify the following code to fit your needs
// This object conforms to what the frontend currently sends.
diff --git a/backend/generator/template/plugin/tasks/api_client.go-template
b/backend/generator/template/plugin/tasks/api_client.go-template
index c56dfd602..6a0d77ffb 100644
--- a/backend/generator/template/plugin/tasks/api_client.go-template
+++ b/backend/generator/template/plugin/tasks/api_client.go-template
@@ -26,7 +26,7 @@ import (
"github.com/apache/incubator-devlake/plugins/{{ .plugin_name }}/models"
core "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/plugins/helper"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
func New{{ .PluginName }}ApiClient(taskCtx plugin.TaskContext, connection
*models.{{ .PluginName }}Connection) (*api.ApiAsyncClient, errors.Error) {
diff --git a/backend/generator/template/plugin/tasks/api_collector.go-template
b/backend/generator/template/plugin/tasks/api_collector.go-template
index eb344436e..5526314ea 100644
--- a/backend/generator/template/plugin/tasks/api_collector.go-template
+++ b/backend/generator/template/plugin/tasks/api_collector.go-template
@@ -25,7 +25,7 @@ import (
core "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/plugins/helper"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
const RAW_{{ .COLLECTOR_DATA_NAME }}_TABLE = "{{ .plugin_name }}_{{
.collector_data_name }}"
@@ -34,21 +34,19 @@ var _ core.SubTaskEntryPoint = Collect{{ .CollectorDataName
}}
func Collect{{ .CollectorDataName }}(taskCtx core.SubTaskContext) errors.Error
{
data := taskCtx.GetData().(*{{ .PluginName }}TaskData)
- iterator, err := helper.NewDateIterator(365)
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_{{
.COLLECTOR_DATA_NAME }}_TABLE)
+ logger := taskCtx.GetLogger()
+
+ collectorWithState, err :=
helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
if err != nil {
return err
}
+ incremental := collectorWithState.IsIncremental()
- collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
- RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: {{ .PluginName }}ApiParams{
- },
- Table: RAW_{{ .COLLECTOR_DATA_NAME }}_TABLE,
- },
+ err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
+ Incremental: incremental,
ApiClient: data.ApiClient,
- Incremental: false,
- Input: iterator,
+ // PageSize: 100,
// TODO write which api would you want request
UrlTemplate: "{{ .HttpPath }}",
Query: func(reqData *helper.RequestData) (url.Values,
errors.Error) {
@@ -66,8 +64,7 @@ func Collect{{ .CollectorDataName }}(taskCtx
core.SubTaskContext) errors.Error {
if err != nil {
return err
}
-
- return collector.Execute()
+ return collectorWithState.Execute()
}
var Collect{{ .CollectorDataName }}Meta = plugin.SubTaskMeta{
diff --git a/backend/generator/template/plugin/tasks/extractor.go-template
b/backend/generator/template/plugin/tasks/extractor.go-template
index 9991a4b19..0457dfaa5 100644
--- a/backend/generator/template/plugin/tasks/extractor.go-template
+++ b/backend/generator/template/plugin/tasks/extractor.go-template
@@ -19,14 +19,13 @@ package tasks
import (
"github.com/apache/incubator-devlake/core/errors"
- core "github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
-var _ core.SubTaskEntryPoint = Extract{{ .ExtractorDataName }}
+var _ plugin.SubTaskEntryPoint = Extract{{ .ExtractorDataName }}
-func Extract{{ .ExtractorDataName }}(taskCtx core.SubTaskContext) errors.Error
{
+func Extract{{ .ExtractorDataName }}(taskCtx plugin.SubTaskContext)
errors.Error {
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Ctx: taskCtx,
diff --git a/backend/plugins/icla/tasks/task_data.go
b/backend/generator/template/plugin/tasks/shared.go-template
similarity index 51%
copy from backend/plugins/icla/tasks/task_data.go
copy to backend/generator/template/plugin/tasks/shared.go-template
index a420a013a..ee2ac613d 100644
--- a/backend/plugins/icla/tasks/task_data.go
+++ b/backend/generator/template/plugin/tasks/shared.go-template
@@ -18,21 +18,24 @@ limitations under the License.
package tasks
import (
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
-type IclaApiParams struct {
-}
-
-type IclaOptions struct {
- // TODO add some custom options here if necessary
- // options means some custom params required by plugin running.
- // Such As How many rows do your want
- // You can use it in sub tasks and you need pass it in main.go and
pipelines.
- Tasks []string `json:"tasks,omitempty"`
-}
-
-type IclaTaskData struct {
- Options *IclaOptions
- ApiClient *helper.ApiAsyncClient
+func CreateRawDataSubTaskArgs(taskCtx plugin.SubTaskContext, rawTable string)
(*api.RawDataSubTaskArgs, *{{ .PluginName }}TaskData) {
+ data := taskCtx.GetData().(*{{ .PluginName }}TaskData)
+ filteredData := *data
+ filteredData.Options = &{{ .PluginName }}Options{}
+ *filteredData.Options = *data.Options
+ var params = {{ .PluginName }}ApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ ProjectKey: data.Options.ProjectKey,
+ HotspotKey: data.Options.HotspotKey,
+ }
+ rawDataSubTaskArgs := &api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: params,
+ Table: rawTable,
+ }
+ return rawDataSubTaskArgs, &filteredData
}
diff --git a/backend/generator/template/plugin/tasks/task_data.go-template
b/backend/generator/template/plugin/tasks/task_data.go-template
index 50fdbd058..fbb0250c3 100644
--- a/backend/generator/template/plugin/tasks/task_data.go-template
+++ b/backend/generator/template/plugin/tasks/task_data.go-template
@@ -24,7 +24,7 @@ type {{ .PluginName }}Options struct {
// TODO add some custom options here if necessary
// options means some custom params required by plugin running.
// Such As How many rows do your want
- // You can use it in sub tasks and you need pass it in main.go and
pipelines.
+ // You can use it in subtasks, and you need to pass it to main.go and
pipelines.
}
type {{ .PluginName }}TaskData struct {
diff --git
a/backend/generator/template/plugin/tasks/task_data_complete_plugin.go-template
b/backend/generator/template/plugin/tasks/task_data_complete_plugin.go-template
index 49f2098ba..dcee683af 100644
---
a/backend/generator/template/plugin/tasks/task_data_complete_plugin.go-template
+++
b/backend/generator/template/plugin/tasks/task_data_complete_plugin.go-template
@@ -20,7 +20,7 @@ package tasks
import (
"fmt"
"github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/plugins/helper"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
type {{ .PluginName }}ApiParams struct {
@@ -30,15 +30,16 @@ type {{ .PluginName }}Options struct {
// TODO add some custom options here if necessary
// options means some custom params required by plugin running.
// Such As How many rows do your want
- // You can use it in sub tasks and you need pass it in main.go and
pipelines.
+ // You can use it in subtasks, and you need to pass it to main.go and
pipelines.
ConnectionId uint64 `json:"connectionId"`
Tasks []string `json:"tasks,omitempty"`
- Since string
+ CreatedDateAfter string `json:"createdDateAfter"
mapstructure:"createdDateAfter,omitempty"`
}
type {{ .PluginName }}TaskData struct {
Options *{{ .PluginName }}Options
ApiClient *api.ApiAsyncClient
+ CreatedDateAfter *time.Time
}
func DecodeAndValidateTaskOptions(options map[string]interface{}) (*{{
.PluginName }}Options, errors.Error) {
@@ -50,4 +51,20 @@ func DecodeAndValidateTaskOptions(options
map[string]interface{}) (*{{ .PluginNa
return nil, errors.Default.New("connectionId is invalid")
}
return &op, nil
-}
\ No newline at end of file
+}
+
+func CreateRawDataSubTaskArgs(taskCtx plugin.SubTaskContext, rawTable string)
(*api.RawDataSubTaskArgs, *{{ .PluginName }}TaskData) {
+ data := taskCtx.GetData().(*{{ .PluginName }}TaskData)
+ filteredData := *data
+ filteredData.Options = &{{ .PluginName }}Options{}
+ *filteredData.Options = *data.Options
+ var params = {{ .PluginName }}ApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ }
+ rawDataSubTaskArgs := &api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: params,
+ Table: rawTable,
+ }
+ return rawDataSubTaskArgs, &filteredData
+}
diff --git a/backend/plugins/icla/tasks/task_data.go
b/backend/plugins/icla/tasks/task_data.go
index a420a013a..dfa35a0ea 100644
--- a/backend/plugins/icla/tasks/task_data.go
+++ b/backend/plugins/icla/tasks/task_data.go
@@ -28,7 +28,7 @@ type IclaOptions struct {
// TODO add some custom options here if necessary
// options means some custom params required by plugin running.
// Such As How many rows do your want
- // You can use it in sub tasks and you need pass it in main.go and
pipelines.
+ // You can use it in subtasks, and you need to pass it to main.go and
pipelines.
Tasks []string `json:"tasks,omitempty"`
}
diff --git a/backend/plugins/sonarqube/api/connection.go
b/backend/plugins/sonarqube/api/connection.go
index 0dea3fcbf..bf518c0ad 100644
--- a/backend/plugins/sonarqube/api/connection.go
+++ b/backend/plugins/sonarqube/api/connection.go
@@ -28,10 +28,14 @@ import (
"time"
)
+type validation struct {
+ Valid bool `json:"valid"`
+}
+
func TestConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
// decode
var err errors.Error
- var connection models.TestConnectionRequest
+ var connection models.SonarqubeConn
if err = api.Decode(input.Body, &connection, vld); err != nil {
return nil, err
}
@@ -40,7 +44,7 @@ func TestConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
context.TODO(),
connection.Endpoint,
map[string]string{
- "Authorization": fmt.Sprintf("%s:", connection.Token),
+ "Authorization": fmt.Sprintf("Basic %s",
connection.GetEncodedToken()),
},
3*time.Second,
connection.Proxy,
@@ -50,13 +54,22 @@ func TestConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
return nil, err
}
- res, err := apiClient.Get("server/version", nil, nil)
+ res, err := apiClient.Get("authentication/validate", nil, nil)
if err != nil {
return nil, err
}
if res.StatusCode != http.StatusOK {
return nil,
errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code: %d",
res.StatusCode))
}
+
+ body := &validation{}
+ err = api.UnmarshalResponse(res, body)
+ if err != nil {
+ return nil, err
+ }
+ if !body.Valid {
+ return nil, errors.Default.New("Authentication failed, please
check your access token.")
+ }
return nil, nil
}
diff --git a/backend/plugins/sonarqube/impl/impl.go
b/backend/plugins/sonarqube/impl/impl.go
index 04679ab82..82b7c55ad 100644
--- a/backend/plugins/sonarqube/impl/impl.go
+++ b/backend/plugins/sonarqube/impl/impl.go
@@ -23,6 +23,7 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "time"
"github.com/apache/incubator-devlake/plugins/sonarqube/api"
"github.com/apache/incubator-devlake/plugins/sonarqube/models"
@@ -50,10 +51,14 @@ func (p Sonarqube) Init(br context.BasicRes) errors.Error {
}
func (p Sonarqube) SubTaskMetas() []plugin.SubTaskMeta {
- return []plugin.SubTaskMeta{}
+ return []plugin.SubTaskMeta{
+ tasks.CollectProjectsMeta,
+ tasks.ExtractProjectsMeta,
+ }
}
func (p Sonarqube) PrepareTaskData(taskCtx plugin.TaskContext, options
map[string]interface{}) (interface{}, errors.Error) {
+ logger := taskCtx.GetLogger()
op, err := tasks.DecodeAndValidateTaskOptions(options)
if err != nil {
return nil, err
@@ -72,11 +77,22 @@ func (p Sonarqube) PrepareTaskData(taskCtx
plugin.TaskContext, options map[strin
if err != nil {
return nil, errors.Default.Wrap(err, "unable to get Sonarqube
API client instance")
}
-
- return &tasks.SonarqubeTaskData{
+ taskData := &tasks.SonarqubeTaskData{
Options: op,
ApiClient: apiClient,
- }, nil
+ }
+ var createdDateAfter time.Time
+ if op.CreatedDateAfter != "" {
+ createdDateAfter, err =
errors.Convert01(time.Parse(time.RFC3339, op.CreatedDateAfter))
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "invalid value
for `createdDateAfter`")
+ }
+ }
+ if !createdDateAfter.IsZero() {
+ taskData.CreatedDateAfter = &createdDateAfter
+ logger.Debug("collect data updated createdDateAfter %s",
createdDateAfter)
+ }
+ return taskData, nil
}
// PkgPath information lost when compiled as plugin(.so)
diff --git a/backend/plugins/sonarqube/models/connection.go
b/backend/plugins/sonarqube/models/connection.go
index fe60b8179..422d88d51 100644
--- a/backend/plugins/sonarqube/models/connection.go
+++ b/backend/plugins/sonarqube/models/connection.go
@@ -17,20 +17,44 @@ limitations under the License.
package models
-import helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+import (
+ "encoding/base64"
+ "fmt"
+ "github.com/apache/incubator-devlake/core/errors"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+
"github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
+ "net/http"
+)
+
+type SonarqubeAccessToken helper.AccessToken
+
+// SetupAuthentication sets up the HTTP Request Authentication
+func (sat SonarqubeAccessToken) SetupAuthentication(req *http.Request)
errors.Error {
+ req.Header.Set("Authorization", fmt.Sprintf("Basic %s",
sat.GetEncodedToken()))
+ return nil
+}
+
+func (sat SonarqubeAccessToken) GetAccessTokenAuthenticator()
apihelperabstract.ApiAuthenticator {
+ return sat
+}
+
+// GetEncodedToken returns encoded bearer token for HTTP Basic Authentication
+func (sat SonarqubeAccessToken) GetEncodedToken() string {
+ return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%v:",
sat.Token)))
+}
// This object conforms to what the frontend currently sends.
type SonarqubeConnection struct {
helper.BaseConnection `mapstructure:",squash"`
helper.RestConnection `mapstructure:",squash"`
// For sonarqube, we can `use user_token:`
- helper.AccessToken `mapstructure:",squash"`
+ SonarqubeAccessToken `mapstructure:",squash"`
}
-type TestConnectionRequest struct {
- Endpoint string `json:"endpoint"`
- Proxy string `json:"proxy"`
- helper.AccessToken `mapstructure:",squash"`
+// SonarqubeConn holds the essential information to connect to the sonarqube
API
+type SonarqubeConn struct {
+ helper.RestConnection `mapstructure:",squash"`
+ SonarqubeAccessToken `mapstructure:",squash"`
}
// This object conforms to what the frontend currently expects.
@@ -40,12 +64,6 @@ type SonarqubeResponse struct {
SonarqubeConnection
}
-// Using User because it requires authentication.
-type ApiUserResponse struct {
- Id int
- Name string `json:"name"`
-}
-
func (SonarqubeConnection) TableName() string {
return "_tool_sonarqube_connections"
}
diff --git
a/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
b/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
index bb3854365..3511113e6 100644
---
a/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
+++
b/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
@@ -30,11 +30,12 @@ func (*addInitTables) Up(basicRes context.BasicRes)
errors.Error {
return migrationhelper.AutoMigrateTables(
basicRes,
&archived.SonarqubeConnection{},
+ &archived.SonarqubeProject{},
)
}
func (*addInitTables) Version() uint64 {
- return 20230111000011
+ return 20230130000011
}
func (*addInitTables) Name() string {
diff --git
a/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_project.go
similarity index 54%
copy from
backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
copy to
backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_project.go
index bb3854365..fa2473dd8 100644
---
a/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
+++
b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_project.go
@@ -15,28 +15,23 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package migrationscripts
+package archived
import (
- "github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/helpers/migrationhelper"
-
"github.com/apache/incubator-devlake/plugins/sonarqube/models/migrationscripts/archived"
+
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+ "time"
)
-type addInitTables struct{}
-
-func (*addInitTables) Up(basicRes context.BasicRes) errors.Error {
- return migrationhelper.AutoMigrateTables(
- basicRes,
- &archived.SonarqubeConnection{},
- )
-}
-
-func (*addInitTables) Version() uint64 {
- return 20230111000011
+type SonarqubeProject struct {
+ archived.NoPKModel
+ Key string `json:"key"
gorm:"type:varchar(64);primaryKey"`
+ Name string `json:"name" gorm:"type:varchar(255)"`
+ Qualifier string `json:"qualifier" gorm:"type:varchar(255)"`
+ Visibility string `json:"visibility" gorm:"type:varchar(64)"`
+ LastAnalysisDate *time.Time `json:"lastAnalysisDate"`
+ Revision string `json:"revision" gorm:"type:varchar(128)"`
}
-func (*addInitTables) Name() string {
- return "sonarqube init schemas"
+func (SonarqubeProject) TableName() string {
+ return "_tool_sonarqube_projects"
}
diff --git
a/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
b/backend/plugins/sonarqube/models/sonarqube_project.go
similarity index 52%
copy from
backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
copy to backend/plugins/sonarqube/models/sonarqube_project.go
index bb3854365..d549aca6b 100644
---
a/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
+++ b/backend/plugins/sonarqube/models/sonarqube_project.go
@@ -15,28 +15,23 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package migrationscripts
+package models
import (
- "github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/helpers/migrationhelper"
-
"github.com/apache/incubator-devlake/plugins/sonarqube/models/migrationscripts/archived"
+ "github.com/apache/incubator-devlake/core/models/common"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
-type addInitTables struct{}
-
-func (*addInitTables) Up(basicRes context.BasicRes) errors.Error {
- return migrationhelper.AutoMigrateTables(
- basicRes,
- &archived.SonarqubeConnection{},
- )
-}
-
-func (*addInitTables) Version() uint64 {
- return 20230111000011
+type SonarqubeProject struct {
+ common.NoPKModel
+ Key string `json:"key"
gorm:"type:varchar(64);primaryKey"`
+ Name string `json:"name" gorm:"type:varchar(255)"`
+ Qualifier string `json:"qualifier"
gorm:"type:varchar(255)"`
+ Visibility string `json:"visibility"
gorm:"type:varchar(64)"`
+ LastAnalysisDate *api.Iso8601Time `json:"lastAnalysisDate"`
+ Revision string `json:"revision"
gorm:"type:varchar(128)"`
}
-func (*addInitTables) Name() string {
- return "sonarqube init schemas"
+func (SonarqubeProject) TableName() string {
+ return "_tool_sonarqube_projects"
}
diff --git a/backend/plugins/sonarqube/sonarqube.go
b/backend/plugins/sonarqube/sonarqube.go
index f5b88d01e..927398a3c 100644
--- a/backend/plugins/sonarqube/sonarqube.go
+++ b/backend/plugins/sonarqube/sonarqube.go
@@ -33,7 +33,7 @@ func main() {
projectName := cmd.Flags().StringP("projectName", "o", "", "sonarqube
projectName")
createdDateAfter := cmd.Flags().StringP("createdDateAfter", "a", "",
"collect data that are created after specified time, ie 2006-05-06T07:08:09Z")
_ = cmd.MarkFlagRequired("connectionId")
- _ = cmd.MarkFlagRequired("projectName")
+ //_ = cmd.MarkFlagRequired("projectName")
cmd.Run = func(cmd *cobra.Command, args []string) {
runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
diff --git a/backend/plugins/sonarqube/tasks/api_client.go
b/backend/plugins/sonarqube/tasks/api_client.go
index 0fff56b32..e7864f4a7 100644
--- a/backend/plugins/sonarqube/tasks/api_client.go
+++ b/backend/plugins/sonarqube/tasks/api_client.go
@@ -32,9 +32,9 @@ import (
func NewSonarqubeApiClient(taskCtx plugin.TaskContext, connection
*models.SonarqubeConnection) (*api.ApiAsyncClient, errors.Error) {
// create synchronize api client so we can calculate api rate limit
dynamically
headers := map[string]string{
- "Authorization": fmt.Sprintf("%s:", connection.Token),
+ "Authorization": fmt.Sprintf("Basic %s",
connection.GetEncodedToken()),
}
- apiClient, err := api.NewApiClient(taskCtx.GetContext(),
connection.Endpoint, headers, 0, connection.Proxy, taskCtx)
+ apiClient, err := api.NewApiClient(taskCtx.GetContext(),
connection.Endpoint, headers, 30*time.Second, connection.Proxy, taskCtx)
if err != nil {
return nil, err
}
diff --git a/backend/plugins/sonarqube/tasks/projects_collector.go
b/backend/plugins/sonarqube/tasks/projects_collector.go
new file mode 100644
index 000000000..da8bd87b4
--- /dev/null
+++ b/backend/plugins/sonarqube/tasks/projects_collector.go
@@ -0,0 +1,81 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "fmt"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "net/http"
+ "net/url"
+)
+
+const RAW_PROJECTS_TABLE = "sonarqube_projects"
+
+var _ plugin.SubTaskEntryPoint = CollectProjects
+
+func CollectProjects(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_PROJECTS_TABLE)
+ logger := taskCtx.GetLogger()
+ logger.Info("collect projects")
+
+ collectorWithState, err :=
helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
+ if err != nil {
+ return err
+ }
+ err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ UrlTemplate: "projects/search",
+ Query: func(reqData *helper.RequestData) (url.Values,
errors.Error) {
+ query := url.Values{}
+ if data.CreatedDateAfter != nil {
+ query.Set("analyzedBefore",
+
data.CreatedDateAfter.Format("2006-01-02"))
+ }
+ if data.Options.ProjectKey != "" {
+ query.Set("q", data.Options.ProjectKey)
+ }
+ query.Set("p", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("ps", fmt.Sprintf("%v", reqData.Pager.Size))
+ return query, nil
+ },
+ GetTotalPages: GetTotalPagesFromResponse,
+ ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
+ var resData struct {
+ Data []json.RawMessage `json:"components"`
+ }
+ err = helper.UnmarshalResponse(res, &resData)
+ return resData.Data, err
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return collectorWithState.Execute()
+}
+
+var CollectProjectsMeta = plugin.SubTaskMeta{
+ Name: "CollectProjects",
+ EntryPoint: CollectProjects,
+ EnabledByDefault: true,
+ Description: "Collect Projects data from Sonarqube api",
+}
diff --git a/backend/plugins/sonarqube/tasks/projects_extractor.go
b/backend/plugins/sonarqube/tasks/projects_extractor.go
new file mode 100644
index 000000000..bd94f5007
--- /dev/null
+++ b/backend/plugins/sonarqube/tasks/projects_extractor.go
@@ -0,0 +1,55 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/sonarqube/models"
+)
+
+var _ plugin.SubTaskEntryPoint = ExtractProjects
+
+func ExtractProjects(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, _ := CreateRawDataSubTaskArgs(taskCtx,
RAW_PROJECTS_TABLE)
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Extract: func(resData *helper.RawData) ([]interface{},
errors.Error) {
+ body := &models.SonarqubeProject{}
+ err := errors.Convert(json.Unmarshal(resData.Data,
body))
+ if err != nil {
+ return nil, err
+ }
+ return []interface{}{body}, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+
+var ExtractProjectsMeta = plugin.SubTaskMeta{
+ Name: "ExtractProjects",
+ EntryPoint: ExtractProjects,
+ EnabledByDefault: true,
+ Description: "Extract raw data into tool layer table
sonarqube_projects",
+}
diff --git a/backend/plugins/sonarqube/tasks/shared.go
b/backend/plugins/sonarqube/tasks/shared.go
new file mode 100644
index 000000000..316ef36c7
--- /dev/null
+++ b/backend/plugins/sonarqube/tasks/shared.go
@@ -0,0 +1,65 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "net/http"
+)
+
+func CreateRawDataSubTaskArgs(taskCtx plugin.SubTaskContext, rawTable string)
(*api.RawDataSubTaskArgs, *SonarqubeTaskData) {
+ data := taskCtx.GetData().(*SonarqubeTaskData)
+ filteredData := *data
+ filteredData.Options = &SonarqubeOptions{}
+ *filteredData.Options = *data.Options
+ var params = SonarqubeApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ ProjectKey: data.Options.ProjectKey,
+ HotspotKey: data.Options.HotspotKey,
+ }
+ rawDataSubTaskArgs := &api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: params,
+ Table: rawTable,
+ }
+ return rawDataSubTaskArgs, &filteredData
+}
+
+func GetTotalPagesFromResponse(res *http.Response, args *api.ApiCollectorArgs)
(int, errors.Error) {
+ body := &SonarqubePagination{}
+ err := api.UnmarshalResponse(res, body)
+ if err != nil {
+ return 0, err
+ }
+ pages := body.Paging.Total / args.PageSize
+ if body.Paging.Total%args.PageSize > 0 {
+ pages++
+ }
+ return pages, nil
+}
+
+type SonarqubePagination struct {
+ Paging Paging `json:"paging"`
+}
+type Paging struct {
+ PageIndex int `json:"pageIndex"`
+ PageSize int `json:"pageSize"`
+ Total int `json:"total"`
+}
diff --git a/backend/plugins/sonarqube/tasks/task_data.go
b/backend/plugins/sonarqube/tasks/task_data.go
index 67dbdbd01..c409340e7 100644
--- a/backend/plugins/sonarqube/tasks/task_data.go
+++ b/backend/plugins/sonarqube/tasks/task_data.go
@@ -20,23 +20,30 @@ package tasks
import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "time"
)
type SonarqubeApiParams struct {
+ ConnectionId uint64 `json:"connectionId"`
+ ProjectKey string
+ HotspotKey string
}
type SonarqubeOptions struct {
// options means some custom params required by plugin running.
// Such As How many rows do your want
- // You can use it in sub tasks and you need pass it in main.go and
pipelines.
- ConnectionId uint64 `json:"connectionId"`
- Tasks []string `json:"tasks,omitempty"`
- Since string
+ // You can use it in subtasks, and you need to pass it to main.go and
pipelines.
+ ConnectionId uint64 `json:"connectionId"`
+ ProjectKey string `json:"projectKey"`
+ HotspotKey string `json:"hotspotKey"`
+ CreatedDateAfter string `json:"createdDateAfter"
mapstructure:"createdDateAfter,omitempty"`
+ Tasks []string `json:"tasks,omitempty"`
}
type SonarqubeTaskData struct {
- Options *SonarqubeOptions
- ApiClient *api.ApiAsyncClient
+ Options *SonarqubeOptions
+ ApiClient *api.ApiAsyncClient
+ CreatedDateAfter *time.Time
}
func DecodeAndValidateTaskOptions(options map[string]interface{})
(*SonarqubeOptions, errors.Error) {
diff --git a/backend/plugins/zentao/tasks/task_data.go
b/backend/plugins/zentao/tasks/task_data.go
index 81b9936da..da07c0a54 100644
--- a/backend/plugins/zentao/tasks/task_data.go
+++ b/backend/plugins/zentao/tasks/task_data.go
@@ -33,7 +33,7 @@ type ZentaoApiParams struct {
type ZentaoOptions struct {
// options means some custom params required by plugin running.
// Such As How many rows do your want
- // You can use it in sub tasks and you need pass it in main.go and
pipelines.
+ // You can use it in subtasks, and you need to pass it to main.go and
pipelines.
ConnectionId uint64 `json:"connectionId"`
ProductId int64
ExecutionId int64
