This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 8c3af5dd7 feat(sonarqube): add issue convertor (#4356)
8c3af5dd7 is described below
commit 8c3af5dd78355f3331beec5422833802063250ec
Author: Warren Chen <[email protected]>
AuthorDate: Wed Feb 8 15:43:25 2023 +0800
feat(sonarqube): add issue convertor (#4356)
---
.../domainlayer/securitytesting/st_issues.go | 54 ++++++++++++++++++++++
.../domainlayer/securitytesting/st_projects.go | 8 ++--
.../20230208_add_security_testing.go | 3 +-
.../models/migrationscripts/archived/st_issues.go | 53 +++++++++++++++++++++
backend/plugins/sonarqube/impl/impl.go | 1 +
.../{project_convertor.go => issues_convertor.go} | 54 ++++++++++++++--------
...{project_convertor.go => projects_convertor.go} | 0
7 files changed, 149 insertions(+), 24 deletions(-)
diff --git a/backend/core/models/domainlayer/securitytesting/st_issues.go
b/backend/core/models/domainlayer/securitytesting/st_issues.go
new file mode 100644
index 000000000..952b33d10
--- /dev/null
+++ b/backend/core/models/domainlayer/securitytesting/st_issues.go
@@ -0,0 +1,54 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package securitytesting
+
+import (
+ "github.com/apache/incubator-devlake/core/models/domainlayer"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+)
+
+type StIssue struct {
+ domainlayer.DomainEntity
+ BatchId string `json:"batchId" gorm:"type:varchar(100)"`
// from collection time
+ Key string `json:"key" gorm:"primaryKey"`
+ Rule string `json:"rule" gorm:"type:varchar(255)"`
+ Severity string `json:"severity" gorm:"type:varchar(255)"`
+ Component string `json:"component"
gorm:"type:varchar(255)"`
+ Project string `json:"project" gorm:"type:varchar(255)"`
+ Line int `json:"line"`
+ Status string `json:"status" gorm:"type:varchar(255)"`
+ Message string `json:"message"`
+ Debt string `json:"debt" gorm:"type:varchar(255)"`
+ Effort string `json:"effort" gorm:"type:varchar(255)"`
+ CommitAuthorEmail string `json:"author"
gorm:"type:varchar(255)"`
+ Assignee string `json:"assignee" gorm:"type:varchar(255)"`
+ Hash string `json:"hash" gorm:"type:varchar(255)"`
+ Tags string `json:"tags" gorm:"type:varchar(255)"`
+ Type string `json:"type" gorm:"type:varchar(255)"`
+ Scope string `json:"scope" gorm:"type:varchar(255)"`
+ StartLine int `json:"startLine"`
+ EndLine int `json:"endLine"`
+ StartOffset int `json:"startOffset"`
+ EndOffset int `json:"endOffset"`
+ CreationDate *api.Iso8601Time `json:"creationDate"`
+ UpdateDate *api.Iso8601Time `json:"updateDate"`
+}
+
+func (StIssue) TableName() string {
+ return "st_issues"
+}
diff --git a/backend/core/models/domainlayer/securitytesting/st_projects.go
b/backend/core/models/domainlayer/securitytesting/st_projects.go
index ce676570f..dc98555aa 100644
--- a/backend/core/models/domainlayer/securitytesting/st_projects.go
+++ b/backend/core/models/domainlayer/securitytesting/st_projects.go
@@ -39,10 +39,10 @@ func (StProject) TableName() string {
return "st_projects"
}
-func (r *StProject) ScopeId() string {
- return r.Id
+func (s *StProject) ScopeId() string {
+ return s.Id
}
-func (r *StProject) ScopeName() string {
- return r.Name
+func (s *StProject) ScopeName() string {
+ return s.Name
}
diff --git
a/backend/core/models/migrationscripts/20230208_add_security_testing.go
b/backend/core/models/migrationscripts/20230208_add_security_testing.go
index e9e2b9db4..eb862fde0 100644
--- a/backend/core/models/migrationscripts/20230208_add_security_testing.go
+++ b/backend/core/models/migrationscripts/20230208_add_security_testing.go
@@ -30,11 +30,12 @@ func (u *addSecurityTesting) Up(basicRes context.BasicRes)
errors.Error {
return migrationhelper.AutoMigrateTables(
basicRes,
&archived.StProject{},
+ &archived.StIssue{},
)
}
func (*addSecurityTesting) Version() uint64 {
- return 20230208000001
+ return 20230208000002
}
func (*addSecurityTesting) Name() string {
diff --git a/backend/core/models/migrationscripts/archived/st_issues.go
b/backend/core/models/migrationscripts/archived/st_issues.go
new file mode 100644
index 000000000..1f163d571
--- /dev/null
+++ b/backend/core/models/migrationscripts/archived/st_issues.go
@@ -0,0 +1,53 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+)
+
+type StIssue struct {
+ DomainEntity
+ BatchId string `json:"batchId"
gorm:"type:varchar(100)"` // from collection time
+ Key string `json:"key" gorm:"primaryKey"`
+ Rule string `json:"rule"
gorm:"type:varchar(255)"`
+ Severity string `json:"severity"
gorm:"type:varchar(255)"`
+ Component string `json:"component"
gorm:"type:varchar(255)"`
+ Project string `json:"project"
gorm:"type:varchar(255)"`
+ Line int `json:"line"`
+ Status string `json:"status"
gorm:"type:varchar(255)"`
+ Message string `json:"message"`
+ Debt string `json:"debt"
gorm:"type:varchar(255)"`
+ Effort string `json:"effort"
gorm:"type:varchar(255)"`
+ CommitAuthorEmail string `json:"author"
gorm:"type:varchar(255)"`
+ Assignee string `json:"assignee"
gorm:"type:varchar(255)"`
+ Hash string `json:"hash"
gorm:"type:varchar(255)"`
+ Tags string `json:"tags"
gorm:"type:varchar(255)"`
+ Type string `json:"type"
gorm:"type:varchar(255)"`
+ Scope string `json:"scope"
gorm:"type:varchar(255)"`
+ StartLine int `json:"startLine"`
+ EndLine int `json:"endLine"`
+ StartOffset int `json:"startOffset"`
+ EndOffset int `json:"endOffset"`
+ CreationDate *api.Iso8601Time `json:"creationDate"`
+ UpdateDate *api.Iso8601Time `json:"updateDate"`
+}
+
+func (StIssue) TableName() string {
+ return "st_issues"
+}
diff --git a/backend/plugins/sonarqube/impl/impl.go
b/backend/plugins/sonarqube/impl/impl.go
index 90e91f503..010baa712 100644
--- a/backend/plugins/sonarqube/impl/impl.go
+++ b/backend/plugins/sonarqube/impl/impl.go
@@ -64,6 +64,7 @@ func (p Sonarqube) SubTaskMetas() []plugin.SubTaskMeta {
tasks.CollectAccountsMeta,
tasks.ExtractAccountsMeta,
tasks.ConvertProjectsMeta,
+ tasks.ConvertIssuesMeta,
}
}
diff --git a/backend/plugins/sonarqube/tasks/project_convertor.go
b/backend/plugins/sonarqube/tasks/issues_convertor.go
similarity index 51%
copy from backend/plugins/sonarqube/tasks/project_convertor.go
copy to backend/plugins/sonarqube/tasks/issues_convertor.go
index aabe498cb..d30a92b9b 100644
--- a/backend/plugins/sonarqube/tasks/project_convertor.go
+++ b/backend/plugins/sonarqube/tasks/issues_convertor.go
@@ -29,42 +29,58 @@ import (
"reflect"
)
-var ConvertProjectsMeta = plugin.SubTaskMeta{
- Name: "convertProjects",
- EntryPoint: ConvertProjects,
+var ConvertIssuesMeta = plugin.SubTaskMeta{
+ Name: "convertIssues",
+ EntryPoint: ConvertIssues,
EnabledByDefault: true,
- Description: "Convert tool layer table sonarqube_projects into
domain layer table projects",
+ Description: "Convert tool layer table sonarqube_issues into
domain layer table issues",
DomainTypes: []string{plugin.DOMAIN_TYPE_SECURITY_TESTING},
}
-func ConvertProjects(taskCtx plugin.SubTaskContext) errors.Error {
+func ConvertIssues(taskCtx plugin.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_PROJECTS_TABLE)
- cursor, err := db.Cursor(dal.From(sonarqubeModels.SonarqubeProject{}),
- dal.Where("connection_id = ? and `key` = ?",
data.Options.ConnectionId, data.Options.ProjectKey))
+ cursor, err := db.Cursor(dal.From(sonarqubeModels.SonarqubeIssue{}),
+ dal.Where("connection_id = ? and project = ?",
data.Options.ConnectionId, data.Options.ProjectKey))
if err != nil {
return err
}
defer cursor.Close()
- accountIdGen :=
didgen.NewDomainIdGenerator(&sonarqubeModels.SonarqubeProject{})
+ accountIdGen :=
didgen.NewDomainIdGenerator(&sonarqubeModels.SonarqubeIssue{})
converter, err := api.NewDataConverter(api.DataConverterArgs{
- InputRowType:
reflect.TypeOf(sonarqubeModels.SonarqubeProject{}),
+ InputRowType:
reflect.TypeOf(sonarqubeModels.SonarqubeIssue{}),
Input: cursor,
RawDataSubTaskArgs: *rawDataSubTaskArgs,
Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
- sonarqubeProject :=
inputRow.(*sonarqubeModels.SonarqubeProject)
- domainProject := &securitytesting.StProject{
- DomainEntity: domainlayer.DomainEntity{Id:
accountIdGen.Generate(data.Options.ConnectionId, sonarqubeProject.Key)},
- Key: sonarqubeProject.Key,
- Name: sonarqubeProject.Name,
- Qualifier: sonarqubeProject.Qualifier,
- Visibility: sonarqubeProject.Visibility,
- LastAnalysisDate:
sonarqubeProject.LastAnalysisDate,
- Revision: sonarqubeProject.Revision,
+ sonarqubeIssue :=
inputRow.(*sonarqubeModels.SonarqubeIssue)
+ domainIssue := &securitytesting.StIssue{
+ DomainEntity: domainlayer.DomainEntity{Id:
accountIdGen.Generate(data.Options.ConnectionId, sonarqubeIssue.Key)},
+ BatchId: sonarqubeIssue.BatchId,
+ Key: sonarqubeIssue.Key,
+ Rule: sonarqubeIssue.Rule,
+ Severity: sonarqubeIssue.Severity,
+ Component: sonarqubeIssue.Component,
+ Project: sonarqubeIssue.Project,
+ Line: sonarqubeIssue.Line,
+ Status: sonarqubeIssue.Status,
+ Message: sonarqubeIssue.Message,
+ Debt: sonarqubeIssue.Debt,
+ Effort: sonarqubeIssue.Effort,
+ CommitAuthorEmail: sonarqubeIssue.Author,
+ Hash: sonarqubeIssue.Hash,
+ Tags: sonarqubeIssue.Tags,
+ Type: sonarqubeIssue.Type,
+ Scope: sonarqubeIssue.Scope,
+ StartLine: sonarqubeIssue.StartLine,
+ EndLine: sonarqubeIssue.EndLine,
+ StartOffset: sonarqubeIssue.StartOffset,
+ EndOffset: sonarqubeIssue.EndOffset,
+ CreationDate: sonarqubeIssue.CreationDate,
+ UpdateDate: sonarqubeIssue.UpdateDate,
}
return []interface{}{
- domainProject,
+ domainIssue,
}, nil
},
})
diff --git a/backend/plugins/sonarqube/tasks/project_convertor.go
b/backend/plugins/sonarqube/tasks/projects_convertor.go
similarity index 100%
rename from backend/plugins/sonarqube/tasks/project_convertor.go
rename to backend/plugins/sonarqube/tasks/projects_convertor.go
