This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 9223ba101 feat: account and issues code block convertor (#4357)
9223ba101 is described below
commit 9223ba1013b7287cdbceb993f12d5c2390930e95
Author: abeizn <[email protected]>
AuthorDate: Wed Feb 8 18:16:49 2023 +0800
feat: account and issues code block convertor (#4357)
* feat: account and issues code block convertor
* feat: account and issues code block convertor
* feat: account and issues code block convertor
* feat: account and issues code block convertor
* feat: account and issues code block convertor
* feat: account and issues code block convertor
---
.../securitytesting/st_issue_code_blocks.go} | 32 ++++-----
.../20230208_add_security_testing.go | 1 +
.../archived/st_issue_code_blocks.go} | 28 ++++----
backend/plugins/gitlab/tasks/account_convertor.go | 2 +-
backend/plugins/sonarqube/impl/impl.go | 1 +
.../archived/sonarqube_issue_code_block.go | 1 +
.../sonarqube/models/sonarqube_issue_code_block.go | 1 +
.../tasks/account_convertor.go | 51 ++++++-------
.../sonarqube/tasks/issue_code_blocks_convertor.go | 84 ++++++++++++++++++++++
.../plugins/sonarqube/tasks/issues_extractor.go | 1 +
10 files changed, 140 insertions(+), 62 deletions(-)
diff --git a/backend/plugins/sonarqube/models/sonarqube_issue_code_block.go
b/backend/core/models/domainlayer/securitytesting/st_issue_code_blocks.go
similarity index 55%
copy from backend/plugins/sonarqube/models/sonarqube_issue_code_block.go
copy to backend/core/models/domainlayer/securitytesting/st_issue_code_blocks.go
index d16939d3e..9d0fbe6e1 100644
--- a/backend/plugins/sonarqube/models/sonarqube_issue_code_block.go
+++ b/backend/core/models/domainlayer/securitytesting/st_issue_code_blocks.go
@@ -15,25 +15,23 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package models
+package securitytesting
-import (
- "github.com/apache/incubator-devlake/core/models/common"
-)
+import "github.com/apache/incubator-devlake/core/models/domainlayer"
-type SonarqubeIssueCodeBlock struct {
- common.NoPKModel
- ConnectionId uint64 `gorm:"primaryKey"`
- Id string `gorm:"primaryKey"`
- IssueKey string `json:"key" gorm:"index"`
- Component string `json:"component" gorm:"index"`
- Msg string `json:"msg" `
- StartLine int `json:"startLine" `
- EndLine int `json:"endLine" `
- StartOffset int `json:"startOffset" `
- EndOffset int `json:"endOffset" `
+type StIssueCodeBlock struct {
+ domainlayer.DomainEntity
+ Id string `gorm:"primaryKey"`
+ IssueKey string `json:"key" gorm:"index"`
+ Component string `json:"component" gorm:"index"`
+ Project string `json:"project" gorm:"index"`
+ Msg string `json:"msg" `
+ StartLine int `json:"startLine" `
+ EndLine int `json:"endLine" `
+ StartOffset int `json:"startOffset" `
+ EndOffset int `json:"endOffset" `
}
-func (SonarqubeIssueCodeBlock) TableName() string {
- return "_tool_sonarqube_issue_code_blocks"
+func (StIssueCodeBlock) TableName() string {
+ return "st_issue_code_blocks"
}
diff --git
a/backend/core/models/migrationscripts/20230208_add_security_testing.go
b/backend/core/models/migrationscripts/20230208_add_security_testing.go
index 481151de9..9ece14f53 100644
--- a/backend/core/models/migrationscripts/20230208_add_security_testing.go
+++ b/backend/core/models/migrationscripts/20230208_add_security_testing.go
@@ -31,6 +31,7 @@ func (u *addSecurityTesting) Up(basicRes context.BasicRes)
errors.Error {
basicRes,
&archived.StProject{},
&archived.StIssue{},
+ &archived.StIssueCodeBlock{},
&archived.StFileMetrics{},
)
}
diff --git
a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_issue_code_block.go
b/backend/core/models/migrationscripts/archived/st_issue_code_blocks.go
similarity index 55%
copy from
backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_issue_code_block.go
copy to backend/core/models/migrationscripts/archived/st_issue_code_blocks.go
index cab175c35..ec36f130d 100644
---
a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_issue_code_block.go
+++ b/backend/core/models/migrationscripts/archived/st_issue_code_blocks.go
@@ -17,21 +17,19 @@ limitations under the License.
package archived
-import
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
-
-type SonarqubeIssueCodeBlock struct {
- ConnectionId uint64 `gorm:"primaryKey"`
- Id string `gorm:"primaryKey"`
- IssueKey string `json:"key" gorm:"index"`
- Component string `json:"component" gorm:"index"`
- Msg string `json:"msg" `
- StartLine int `json:"startLine" `
- EndLine int `json:"endLine" `
- StartOffset int `json:"startOffset" `
- EndOffset int `json:"endOffset" `
- archived.NoPKModel
+type StIssueCodeBlock struct {
+ DomainEntity
+ Id string `gorm:"primaryKey"`
+ IssueKey string `json:"key" gorm:"index"`
+ Component string `json:"component" gorm:"index"`
+ Project string `json:"project" gorm:"index"`
+ Msg string `json:"msg" `
+ StartLine int `json:"startLine" `
+ EndLine int `json:"endLine" `
+ StartOffset int `json:"startOffset" `
+ EndOffset int `json:"endOffset" `
}
-func (SonarqubeIssueCodeBlock) TableName() string {
- return "_tool_sonarqube_issue_code_blocks"
+func (StIssueCodeBlock) TableName() string {
+ return "st_issue_code_blocks"
}
diff --git a/backend/plugins/gitlab/tasks/account_convertor.go
b/backend/plugins/gitlab/tasks/account_convertor.go
index f03a5f42e..2a5799979 100644
--- a/backend/plugins/gitlab/tasks/account_convertor.go
+++ b/backend/plugins/gitlab/tasks/account_convertor.go
@@ -33,7 +33,7 @@ var ConvertAccountsMeta = plugin.SubTaskMeta{
Name: "convertAccounts",
EntryPoint: ConvertAccounts,
EnabledByDefault: true,
- Description: "Convert tool layer table gitlab_users into domain
layer table users",
+ Description: "Convert tool layer table gitlab_users into domain
layer table accounts",
DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
}
diff --git a/backend/plugins/sonarqube/impl/impl.go
b/backend/plugins/sonarqube/impl/impl.go
index 63d60f6ff..b9412d891 100644
--- a/backend/plugins/sonarqube/impl/impl.go
+++ b/backend/plugins/sonarqube/impl/impl.go
@@ -65,6 +65,7 @@ func (p Sonarqube) SubTaskMetas() []plugin.SubTaskMeta {
tasks.ExtractAccountsMeta,
tasks.ConvertProjectsMeta,
tasks.ConvertIssuesMeta,
+ tasks.ConvertIssueCodeBlocksMeta,
tasks.ConvertHotspotsMeta,
tasks.ConvertFileMetricsMeta,
}
diff --git
a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_issue_code_block.go
b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_issue_code_block.go
index cab175c35..57aa332ba 100644
---
a/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_issue_code_block.go
+++
b/backend/plugins/sonarqube/models/migrationscripts/archived/sonarqube_issue_code_block.go
@@ -24,6 +24,7 @@ type SonarqubeIssueCodeBlock struct {
Id string `gorm:"primaryKey"`
IssueKey string `json:"key" gorm:"index"`
Component string `json:"component" gorm:"index"`
+ Project string `json:"project" gorm:"index"`
Msg string `json:"msg" `
StartLine int `json:"startLine" `
EndLine int `json:"endLine" `
diff --git a/backend/plugins/sonarqube/models/sonarqube_issue_code_block.go
b/backend/plugins/sonarqube/models/sonarqube_issue_code_block.go
index d16939d3e..7e5ed3fbb 100644
--- a/backend/plugins/sonarqube/models/sonarqube_issue_code_block.go
+++ b/backend/plugins/sonarqube/models/sonarqube_issue_code_block.go
@@ -27,6 +27,7 @@ type SonarqubeIssueCodeBlock struct {
Id string `gorm:"primaryKey"`
IssueKey string `json:"key" gorm:"index"`
Component string `json:"component" gorm:"index"`
+ Project string `json:"project" gorm:"index"`
Msg string `json:"msg" `
StartLine int `json:"startLine" `
EndLine int `json:"endLine" `
diff --git a/backend/plugins/gitlab/tasks/account_convertor.go
b/backend/plugins/sonarqube/tasks/account_convertor.go
similarity index 66%
copy from backend/plugins/gitlab/tasks/account_convertor.go
copy to backend/plugins/sonarqube/tasks/account_convertor.go
index f03a5f42e..8ed5a439f 100644
--- a/backend/plugins/gitlab/tasks/account_convertor.go
+++ b/backend/plugins/sonarqube/tasks/account_convertor.go
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -25,48 +27,31 @@ import (
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- gitlabModels "github.com/apache/incubator-devlake/plugins/gitlab/models"
- "reflect"
+ "github.com/apache/incubator-devlake/plugins/sonarqube/models"
)
-var ConvertAccountsMeta = plugin.SubTaskMeta{
- Name: "convertAccounts",
- EntryPoint: ConvertAccounts,
- EnabledByDefault: true,
- Description: "Convert tool layer table gitlab_users into domain
layer table users",
- DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
-}
-
func ConvertAccounts(taskCtx plugin.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
- data := taskCtx.GetData().(*GitlabTaskData)
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_ACCOUNTS_TABLE)
- cursor, err := db.Cursor(dal.From(gitlabModels.GitlabAccount{}),
dal.Where("connection_id = ?", data.Options.ConnectionId))
+ cursor, err := db.Cursor(dal.From(&models.SonarqubeAccount{}),
dal.Where("connection_id = ?", data.Options.ConnectionId))
if err != nil {
return err
}
defer cursor.Close()
- accountIdGen :=
didgen.NewDomainIdGenerator(&gitlabModels.GitlabAccount{})
+ accountIdGen := didgen.NewDomainIdGenerator(&models.SonarqubeAccount{})
converter, err := api.NewDataConverter(api.DataConverterArgs{
- InputRowType: reflect.TypeOf(gitlabModels.GitlabAccount{}),
- Input: cursor,
- RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: GitlabApiParams{
- ConnectionId: data.Options.ConnectionId,
- ProjectId: data.Options.ProjectId,
- },
- Table: RAW_USER_TABLE,
- },
+ InputRowType: reflect.TypeOf(models.SonarqubeAccount{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
- GitlabAccount := inputRow.(*gitlabModels.GitlabAccount)
+ SonarqubeAccount := inputRow.(*models.SonarqubeAccount)
domainUser := &crossdomain.Account{
- DomainEntity: domainlayer.DomainEntity{Id:
accountIdGen.Generate(data.Options.ConnectionId, GitlabAccount.GitlabId)},
- UserName: GitlabAccount.Username,
- FullName: GitlabAccount.Name,
- Email: GitlabAccount.Email,
- AvatarUrl: GitlabAccount.AvatarUrl,
+ DomainEntity: domainlayer.DomainEntity{Id:
accountIdGen.Generate(data.Options.ConnectionId, SonarqubeAccount.Login)},
+ UserName: SonarqubeAccount.Login,
+ FullName: SonarqubeAccount.Name,
+ Email: SonarqubeAccount.Email,
}
return []interface{}{
@@ -81,3 +66,11 @@ func ConvertAccounts(taskCtx plugin.SubTaskContext)
errors.Error {
return converter.Execute()
}
+
+var ConvertAccountsMeta = plugin.SubTaskMeta{
+ Name: "convertAccounts",
+ EntryPoint: ConvertAccounts,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table sonarqube_accounts into
domain layer table accounts",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
+}
diff --git a/backend/plugins/sonarqube/tasks/issue_code_blocks_convertor.go
b/backend/plugins/sonarqube/tasks/issue_code_blocks_convertor.go
new file mode 100644
index 000000000..36db485d8
--- /dev/null
+++ b/backend/plugins/sonarqube/tasks/issue_code_blocks_convertor.go
@@ -0,0 +1,84 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+
"github.com/apache/incubator-devlake/core/models/domainlayer/securitytesting"
+
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/domainlayer"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/sonarqube/models"
+)
+
+func ConvertIssueCodeBlocks(taskCtx plugin.SubTaskContext) errors.Error {
+ db := taskCtx.GetDal()
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_ISSUES_TABLE)
+
+ cursor, err := db.Cursor(dal.From(&models.SonarqubeIssueCodeBlock{}),
dal.Where("connection_id = ? and project = ? ",
+ data.Options.ConnectionId, data.Options.ProjectKey))
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ idGen := didgen.NewDomainIdGenerator(&models.SonarqubeIssueCodeBlock{})
+ converter, err := api.NewDataConverter(api.DataConverterArgs{
+ InputRowType:
reflect.TypeOf(models.SonarqubeIssueCodeBlock{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
+ sonarqubeIssueCodeBlock :=
inputRow.(*models.SonarqubeIssueCodeBlock)
+ domainIssueCodeBlock :=
&securitytesting.StIssueCodeBlock{
+ DomainEntity: domainlayer.DomainEntity{Id:
idGen.Generate(data.Options.ConnectionId, sonarqubeIssueCodeBlock.Id)},
+ Id: sonarqubeIssueCodeBlock.Id,
+ IssueKey: sonarqubeIssueCodeBlock.IssueKey,
+ Component: sonarqubeIssueCodeBlock.Component,
+ Msg: sonarqubeIssueCodeBlock.Msg,
+ Project: sonarqubeIssueCodeBlock.Project,
+ StartLine: sonarqubeIssueCodeBlock.StartLine,
+ EndLine: sonarqubeIssueCodeBlock.EndLine,
+ StartOffset:
sonarqubeIssueCodeBlock.StartOffset,
+ EndOffset: sonarqubeIssueCodeBlock.EndOffset,
+ }
+
+ return []interface{}{
+ domainIssueCodeBlock,
+ }, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
+
+var ConvertIssueCodeBlocksMeta = plugin.SubTaskMeta{
+ Name: "convertIssueCodeBlocks",
+ EntryPoint: ConvertIssueCodeBlocks,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table sonarqube_issues into
domain layer table st_issue_code_blocks",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_SECURITY_TESTING},
+}
diff --git a/backend/plugins/sonarqube/tasks/issues_extractor.go
b/backend/plugins/sonarqube/tasks/issues_extractor.go
index 97e941797..e8d4eedc8 100644
--- a/backend/plugins/sonarqube/tasks/issues_extractor.go
+++ b/backend/plugins/sonarqube/tasks/issues_extractor.go
@@ -82,6 +82,7 @@ func ExtractIssues(taskCtx plugin.SubTaskContext)
errors.Error {
ConnectionId:
data.Options.ConnectionId,
IssueKey:
sonarqubeIssue.Key,
Component:
location.Component,
+ Project: body.Project,
Msg: location.Msg,
StartLine:
location.TextRange.StartLine,
EndLine:
location.TextRange.EndLine,
