This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 395b4f5ce feat(sonarqube): add file metrics (#4342)
395b4f5ce is described below
commit 395b4f5ceac9870303a256220d453068b51badf1
Author: Warren Chen <[email protected]>
AuthorDate: Tue Feb 7 15:58:03 2023 +0800
feat(sonarqube): add file metrics (#4342)
---
.../plugin/tasks/api_collector.go-template | 1 -
.../template/plugin/tasks/extractor.go-template | 10 +-
backend/plugins/sonarqube/impl/impl.go | 2 +
.../migrationscripts/20230111_add_init_tables.go | 3 +-
.../archived/sonarqube_file_metrics.go | 51 +++++++
.../sonarqube/models/sonarqube_file_metrics.go | 51 +++++++
.../sonarqube/tasks/filemetrics_collector.go} | 50 +++----
.../sonarqube/tasks/filemetrics_extractor.go | 152 +++++++++++++++++++++
.../plugins/sonarqube/tasks/hotspots_extractor.go | 4 +-
9 files changed, 289 insertions(+), 35 deletions(-)
diff --git a/backend/generator/template/plugin/tasks/api_collector.go-template
b/backend/generator/template/plugin/tasks/api_collector.go-template
index 4199cd8b9..3626fe4e5 100644
--- a/backend/generator/template/plugin/tasks/api_collector.go-template
+++ b/backend/generator/template/plugin/tasks/api_collector.go-template
@@ -33,7 +33,6 @@ const RAW_{{ .COLLECTOR_DATA_NAME }}_TABLE = "{{ .plugin_name
}}_{{ .collector_d
var _ plugin.SubTaskEntryPoint = Collect{{ .CollectorDataName }}
func Collect{{ .CollectorDataName }}(taskCtx plugin.SubTaskContext)
errors.Error {
- data := taskCtx.GetData().(*{{ .PluginName }}TaskData)
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_{{
.COLLECTOR_DATA_NAME }}_TABLE)
logger := taskCtx.GetLogger()
diff --git a/backend/generator/template/plugin/tasks/extractor.go-template
b/backend/generator/template/plugin/tasks/extractor.go-template
index 0457dfaa5..73068a5dd 100644
--- a/backend/generator/template/plugin/tasks/extractor.go-template
+++ b/backend/generator/template/plugin/tasks/extractor.go-template
@@ -26,13 +26,11 @@ import (
var _ plugin.SubTaskEntryPoint = Extract{{ .ExtractorDataName }}
func Extract{{ .ExtractorDataName }}(taskCtx plugin.SubTaskContext)
errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_{{
.COLLECTOR_DATA_NAME }}_TABLE)
+
extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
- RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: {{ .PluginName }}ApiParams{
- },
- Table: RAW_{{ .COLLECTOR_DATA_NAME }}_TABLE,
- },
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+
Extract: func(resData *helper.RawData) ([]interface{},
errors.Error) {
extractedModels := make([]interface{}, 0)
println(resData.Data)
diff --git a/backend/plugins/sonarqube/impl/impl.go
b/backend/plugins/sonarqube/impl/impl.go
index aaaff7872..aba0c7785 100644
--- a/backend/plugins/sonarqube/impl/impl.go
+++ b/backend/plugins/sonarqube/impl/impl.go
@@ -59,6 +59,8 @@ func (p Sonarqube) SubTaskMetas() []plugin.SubTaskMeta {
tasks.ExtractIssuesMeta,
tasks.CollectHotspotsMeta,
tasks.ExtractHotspotsMeta,
+ tasks.CollectFilemetricsMeta,
+ tasks.ExtractFilemetricsMeta,
}
}
diff --git
a/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
b/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
index 90c19cc7b..1aaf21f04 100644
---
a/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
+++
b/backend/plugins/sonarqube/models/migrationscripts/20230111_add_init_tables.go
@@ -33,12 +33,13 @@ func (*addInitTables) Up(basicRes context.BasicRes)
errors.Error {
&archived.SonarqubeProject{},
&archived.SonarqubeHotspot{},
&archived.SonarqubeIssue{},
+ &archived.SonarqubeFileMetrics{},
&archived.SonarqubeIssueCodeBlock{},
)
}
func (*addInitTables) Version() uint64 {
- return 20230206200015
+ return 20230206200021
}
func (*addInitTables) Name() string {
diff --git
a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_file_metrics.go
b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_file_metrics.go
new file mode 100644
index 000000000..9c2b3f0fd
--- /dev/null
+++
b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_file_metrics.go
@@ -0,0 +1,51 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+)
+
+type SonarqubeFileMetrics struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ ComponentKey string `json:"component_key"
gorm:"primaryKey"`
+ FileName string `json:"file_name"`
+ FilePath string `json:"file_path"`
+ FileLanguage string `json:"file_language"`
+ BatchID string `json:"batch_id"`
+ SqaleIndex string `json:"sqale_index"`
+ SqaleRating string `json:"sqale_rating"`
+ ReliabilityRating string `json:"reliability_rating"`
+ SecurityRating string `json:"security_rating"`
+ SecurityReviewRating string `json:"security_review_rating"`
+ Ncloc int `json:"ncloc"`
+ DuplicatedBlocks int `json:"duplicated_blocks"`
+ DuplicatedLinesDensity float64 `json:"duplicated_lines_density"`
+ CodeSmells int `json:"code_smells"`
+ Bugs int `json:"bugs"`
+ Vulnerabilities int `json:"vulnerabilities"`
+ SecurityHotspots int `json:"security_hotspots"`
+ SecurityHotspotsReviewed float64 `json:"security_hotspots_reviewed"`
+ Coverage float64 `json:"coverage"`
+ LinesToCover int `json:"lines_to_cover"`
+ archived.NoPKModel
+}
+
+func (SonarqubeFileMetrics) TableName() string {
+ return "_tool_sonarqube_file_metrics"
+}
diff --git a/backend/plugins/sonarqube/models/sonarqube_file_metrics.go
b/backend/plugins/sonarqube/models/sonarqube_file_metrics.go
new file mode 100644
index 000000000..0864d1266
--- /dev/null
+++ b/backend/plugins/sonarqube/models/sonarqube_file_metrics.go
@@ -0,0 +1,51 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/core/models/common"
+)
+
+type SonarqubeFileMetrics struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ ComponentKey string `json:"component_key"
gorm:"primaryKey"`
+ FileName string `json:"file_name"`
+ FilePath string `json:"file_path"`
+ FileLanguage string `json:"file_language"`
+ BatchID string `json:"batch_id"`
+ SqaleIndex string `json:"sqale_index"`
+ SqaleRating string `json:"sqale_rating"`
+ ReliabilityRating string `json:"reliability_rating"`
+ SecurityRating string `json:"security_rating"`
+ SecurityReviewRating string `json:"security_review_rating"`
+ Ncloc int `json:"ncloc"`
+ DuplicatedBlocks int `json:"duplicated_blocks"`
+ DuplicatedLinesDensity float64 `json:"duplicated_lines_density"`
+ CodeSmells int `json:"code_smells"`
+ Bugs int `json:"bugs"`
+ Vulnerabilities int `json:"vulnerabilities"`
+ SecurityHotspots int `json:"security_hotspots"`
+ SecurityHotspotsReviewed float64 `json:"security_hotspots_reviewed"`
+ Coverage float64 `json:"coverage"`
+ LinesToCover int `json:"lines_to_cover"`
+ common.NoPKModel
+}
+
+func (SonarqubeFileMetrics) TableName() string {
+ return "_tool_sonarqube_file_metrics"
+}
diff --git a/backend/generator/template/plugin/tasks/api_collector.go-template
b/backend/plugins/sonarqube/tasks/filemetrics_collector.go
similarity index 57%
copy from backend/generator/template/plugin/tasks/api_collector.go-template
copy to backend/plugins/sonarqube/tasks/filemetrics_collector.go
index 4199cd8b9..bf70a0b42 100644
--- a/backend/generator/template/plugin/tasks/api_collector.go-template
+++ b/backend/plugins/sonarqube/tasks/filemetrics_collector.go
@@ -19,25 +19,21 @@ package tasks
import (
"encoding/json"
- "net/http"
- "net/url"
- "strconv"
-
- "github.com/apache/incubator-devlake/core/plugin"
+ "fmt"
"github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "net/http"
+ "net/url"
)
-const RAW_{{ .COLLECTOR_DATA_NAME }}_TABLE = "{{ .plugin_name }}_{{
.collector_data_name }}"
-
-var _ plugin.SubTaskEntryPoint = Collect{{ .CollectorDataName }}
+const RAW_FILEMETRICS_TABLE = "sonarqube_filemetrics"
-func Collect{{ .CollectorDataName }}(taskCtx plugin.SubTaskContext)
errors.Error {
- data := taskCtx.GetData().(*{{ .PluginName }}TaskData)
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_{{
.COLLECTOR_DATA_NAME }}_TABLE)
- logger := taskCtx.GetLogger()
+var _ plugin.SubTaskEntryPoint = CollectFilemetrics
- collectorWithState, err :=
helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
+func CollectFilemetrics(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_FILEMETRICS_TABLE)
+ collectorWithState, err :=
helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
if err != nil {
return err
}
@@ -46,19 +42,23 @@ func Collect{{ .CollectorDataName }}(taskCtx
plugin.SubTaskContext) errors.Error
err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
Incremental: incremental,
ApiClient: data.ApiClient,
- // PageSize: 100,
- // TODO write which api would you want request
- UrlTemplate: "{{ .HttpPath }}",
+ PageSize: 100,
+ UrlTemplate: "measures/component_tree",
Query: func(reqData *helper.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
- input := reqData.Input.(*helper.DatePair)
- query.Set("start_time",
strconv.FormatInt(input.PairStartTime.Unix(), 10))
- query.Set("end_time",
strconv.FormatInt(input.PairEndTime.Unix(), 10))
+ query.Set("component", data.Options.ProjectKey)
+ query.Set("qualifiers", "FIL")
+ query.Set("metricKeys",
"code_smells,sqale_index,sqale_rating,bugs,reliability_rating,vulnerabilities,security_rating,security_hotspots,security_hotspots_reviewed,security_review_rating,ncloc,coverage,lines_to_cover,duplicated_lines_density,duplicated_blocks")
+ query.Set("p", fmt.Sprintf("%v", reqData.Pager.Page))
+ query.Set("ps", fmt.Sprintf("%v", reqData.Pager.Size))
return query, nil
},
ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
- // TODO decode result from api request
- return []json.RawMessage{}, nil
+ var resData struct {
+ Data []json.RawMessage `json:"components"`
+ }
+ err = helper.UnmarshalResponse(res, &resData)
+ return resData.Data, err
},
})
if err != nil {
@@ -67,9 +67,9 @@ func Collect{{ .CollectorDataName }}(taskCtx
plugin.SubTaskContext) errors.Error
return collectorWithState.Execute()
}
-var Collect{{ .CollectorDataName }}Meta = plugin.SubTaskMeta{
- Name: "Collect{{ .CollectorDataName }}",
- EntryPoint: Collect{{ .CollectorDataName }},
+var CollectFilemetricsMeta = plugin.SubTaskMeta{
+ Name: "CollectFilemetrics",
+ EntryPoint: CollectFilemetrics,
EnabledByDefault: true,
- Description: "Collect {{ .CollectorDataName }} data from {{
.PluginName }} api",
+ Description: "Collect Filemetrics data from Sonarqube api",
}
diff --git a/backend/plugins/sonarqube/tasks/filemetrics_extractor.go
b/backend/plugins/sonarqube/tasks/filemetrics_extractor.go
new file mode 100644
index 000000000..7f6744cc4
--- /dev/null
+++ b/backend/plugins/sonarqube/tasks/filemetrics_extractor.go
@@ -0,0 +1,152 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/sonarqube/models"
+ "strconv"
+)
+
+var _ plugin.SubTaskEntryPoint = ExtractFilemetrics
+
+func ExtractFilemetrics(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_FILEMETRICS_TABLE)
+
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+
+ Extract: func(resData *helper.RawData) ([]interface{},
errors.Error) {
+ body := &fileMetricsResponse{}
+ err := errors.Convert(json.Unmarshal(resData.Data,
body))
+ if err != nil {
+ return nil, err
+ }
+ fileMetrics := &models.SonarqubeFileMetrics{
+ ConnectionId: data.Options.ConnectionId,
+ ComponentKey: body.Key,
+ FileName: body.Name,
+ FilePath: body.Path,
+ FileLanguage: body.Language,
+ //BatchID: "",
+ }
+ alphabetMap := map[string]string{
+ "1.0": "A",
+ "2.0": "B",
+ "3.0": "C",
+ "4.0": "D",
+ "5.0": "E",
+ }
+ for _, v := range body.Measures {
+ switch v.Metric {
+ case "sqale_index":
+ fileMetrics.SqaleIndex = v.Value
+ case "sqale_rating":
+ fileMetrics.SqaleRating = v.Value
+ case "reliability_rating":
+ fileMetrics.ReliabilityRating =
alphabetMap[v.Value]
+ case "security_rating":
+ fileMetrics.SecurityRating =
alphabetMap[v.Value]
+ case "security_review_rating":
+ fileMetrics.SecurityReviewRating =
alphabetMap[v.Value]
+ case "ncloc":
+ fileMetrics.Ncloc, err =
errors.Convert01(strconv.Atoi(v.Value))
+ if err != nil {
+ return nil, err
+ }
+ case "duplicated_blocks":
+ fileMetrics.DuplicatedBlocks, err =
errors.Convert01(strconv.Atoi(v.Value))
+ if err != nil {
+ return nil, err
+ }
+
+ case "duplicated_lines_density":
+ fileMetrics.DuplicatedLinesDensity, err
= errors.Convert01(strconv.ParseFloat(v.Value, 32))
+ if err != nil {
+ return nil, err
+ }
+ case "code_smells":
+ fileMetrics.CodeSmells, err =
errors.Convert01(strconv.Atoi(v.Value))
+ if err != nil {
+ return nil, err
+ }
+ case "bugs":
+ fileMetrics.Bugs, err =
errors.Convert01(strconv.Atoi(v.Value))
+ if err != nil {
+ return nil, err
+ }
+ case "vulnerabilities":
+ fileMetrics.Vulnerabilities, err =
errors.Convert01(strconv.Atoi(v.Value))
+ if err != nil {
+ return nil, err
+ }
+ case "security_hotspots":
+ fileMetrics.SecurityHotspots, err =
errors.Convert01(strconv.Atoi(v.Value))
+ if err != nil {
+ return nil, err
+ }
+ case "security_hotspots_reviewed":
+ fileMetrics.SecurityHotspotsReviewed,
err = errors.Convert01(strconv.ParseFloat(v.Value, 32))
+ if err != nil {
+ return nil, err
+ }
+ case "coverage":
+ fileMetrics.Coverage, err =
errors.Convert01(strconv.ParseFloat(v.Value, 32))
+ if err != nil {
+ return nil, err
+ }
+ case "lines_to_cover":
+ fileMetrics.LinesToCover, err =
errors.Convert01(strconv.Atoi(v.Value))
+ if err != nil {
+ return nil, err
+ }
+ }
+ }
+ return []interface{}{fileMetrics}, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+
+var ExtractFilemetricsMeta = plugin.SubTaskMeta{
+ Name: "ExtractFilemetrics",
+ EntryPoint: ExtractFilemetrics,
+ EnabledByDefault: true,
+ Description: "Extract raw data into tool layer table
sonarqube_filemetrics",
+}
+
+type fileMetricsResponse struct {
+ Key string `json:"key"`
+ Name string `json:"name"`
+ Qualifier string `json:"qualifier"`
+ Path string `json:"path"`
+ Language string `json:"language"`
+ Measures []Measure `json:"measures"`
+}
+type Measure struct {
+ Metric string `json:"metric"`
+ Value string `json:"value"`
+ BestValue bool `json:"bestValue,omitempty"`
+}
diff --git a/backend/plugins/sonarqube/tasks/hotspots_extractor.go
b/backend/plugins/sonarqube/tasks/hotspots_extractor.go
index 22d4ea440..0ba42f744 100644
--- a/backend/plugins/sonarqube/tasks/hotspots_extractor.go
+++ b/backend/plugins/sonarqube/tasks/hotspots_extractor.go
@@ -35,11 +35,11 @@ func ExtractHotspots(taskCtx plugin.SubTaskContext)
errors.Error {
Extract: func(resData *helper.RawData) ([]interface{},
errors.Error) {
body := &models.SonarqubeHotspot{}
err := errors.Convert(json.Unmarshal(resData.Data,
body))
- body.ConnectionId = data.Options.ConnectionId
- //body.BatchId = ""
if err != nil {
return nil, err
}
+ body.ConnectionId = data.Options.ConnectionId
+ //body.BatchId = ""
return []interface{}{body}, nil
},
})
