This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new ec7ffffb1 fix(sonarqube): fix data loss (#4477)
ec7ffffb1 is described below
commit ec7ffffb1bf3ae8406631fb39198d001525c8127
Author: Warren Chen <[email protected]>
AuthorDate: Tue Feb 21 22:11:02 2023 +0800
fix(sonarqube): fix data loss (#4477)
---
backend/plugins/sonarqube/impl/impl.go | 38 +++++++++-------------
.../plugins/sonarqube/tasks/issues_collector.go | 1 -
.../plugins/sonarqube/tasks/issues_extractor.go | 7 ++--
devops/releases/lake-v0.14.0/docker-compose.yml | 3 ++
devops/releases/lake-v0.15.0/docker-compose.yml | 4 +++
docker-compose.yml | 4 +++
6 files changed, 28 insertions(+), 29 deletions(-)
diff --git a/backend/plugins/sonarqube/impl/impl.go
b/backend/plugins/sonarqube/impl/impl.go
index e4655274d..c3be0ba08 100644
--- a/backend/plugins/sonarqube/impl/impl.go
+++ b/backend/plugins/sonarqube/impl/impl.go
@@ -123,30 +123,22 @@ func (p Sonarqube) PrepareTaskData(taskCtx
plugin.TaskContext, options map[strin
Options: op,
ApiClient: apiClient,
}
- if op.ProjectKey != "" {
- var scope *models.SonarqubeProject
- var apiProject *tasks.SonarqubeApiProject
- // support v100 & advance mode
- // If we still cannot find the record in db, we have to request
from remote server and save it to db
- db := taskCtx.GetDal()
- err = db.First(&scope, dal.Where("connection_id = ? AND
project_key = ?", op.ConnectionId, op.ProjectKey))
- if err != nil && db.IsErrorNotFound(err) {
- apiProject, err = api.GetApiProject(op.ProjectKey,
apiClient)
- if err != nil {
- return nil, err
- }
- logger.Debug(fmt.Sprintf("Current project: %s",
apiProject.ProjectKey))
- scope = tasks.ConvertProject(apiProject)
- scope.ConnectionId = op.ConnectionId
- err = taskCtx.GetDal().CreateIfNotExist(&scope)
- if err != nil {
- return nil, err
- }
- }
- if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail
to find project: %s", op.ProjectKey))
- }
+ // even we have project in _tool_sonaqube_projects, we still need to
collect project to update LastAnalysisDate
+ var scope *models.SonarqubeProject
+ var apiProject *tasks.SonarqubeApiProject
+ apiProject, err = api.GetApiProject(op.ProjectKey, apiClient)
+ if err != nil {
+ return nil, err
}
+ logger.Debug(fmt.Sprintf("Current project: %s", apiProject.ProjectKey))
+ scope = tasks.ConvertProject(apiProject)
+ scope.ConnectionId = op.ConnectionId
+ err = taskCtx.GetDal().CreateOrUpdate(&scope)
+ if err != nil {
+ return nil, err
+ }
+ taskData.LastAnalysisDate = scope.LastAnalysisDate.ToNullableTime()
+
var createdDateAfter time.Time
if op.CreatedDateAfter != "" {
createdDateAfter, err =
errors.Convert01(time.Parse(time.RFC3339, op.CreatedDateAfter))
diff --git a/backend/plugins/sonarqube/tasks/issues_collector.go
b/backend/plugins/sonarqube/tasks/issues_collector.go
index 4c12a4fb3..8e937e7d3 100644
--- a/backend/plugins/sonarqube/tasks/issues_collector.go
+++ b/backend/plugins/sonarqube/tasks/issues_collector.go
@@ -35,7 +35,6 @@ func CollectIssues(taskCtx plugin.SubTaskContext) (err
errors.Error) {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_ISSUES_TABLE)
logger := taskCtx.GetLogger()
logger.Info("collect issues")
-
collectorWithState, err :=
helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
if err != nil {
return err
diff --git a/backend/plugins/sonarqube/tasks/issues_extractor.go
b/backend/plugins/sonarqube/tasks/issues_extractor.go
index 2670910ad..3facf5ead 100644
--- a/backend/plugins/sonarqube/tasks/issues_extractor.go
+++ b/backend/plugins/sonarqube/tasks/issues_extractor.go
@@ -22,13 +22,12 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
- "hash"
- "strings"
-
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/sonarqube/models"
+ "hash"
+ "strings"
)
var _ plugin.SubTaskEntryPoint = ExtractIssues
@@ -45,7 +44,6 @@ func ExtractIssues(taskCtx plugin.SubTaskContext)
errors.Error {
if err != nil {
return nil, err
}
-
sonarqubeIssue := &models.SonarqubeIssue{
ConnectionId: data.Options.ConnectionId,
IssueKey: body.Key,
@@ -91,7 +89,6 @@ func ExtractIssues(taskCtx plugin.SubTaskContext)
errors.Error {
results = append(results, codeBlock)
}
}
-
return results, nil
},
})
diff --git a/devops/releases/lake-v0.14.0/docker-compose.yml
b/devops/releases/lake-v0.14.0/docker-compose.yml
index f912a4307..e9b4d79b9 100644
--- a/devops/releases/lake-v0.14.0/docker-compose.yml
+++ b/devops/releases/lake-v0.14.0/docker-compose.yml
@@ -27,6 +27,9 @@ services:
MYSQL_DATABASE: lake
MYSQL_USER: merico
MYSQL_PASSWORD: merico
+ command:
+ --character-set-server=utf8mb4
+ --collation-server=utf8mb4_bin
grafana:
image: apache/devlake-dashboard:v0.14.0
diff --git a/devops/releases/lake-v0.15.0/docker-compose.yml
b/devops/releases/lake-v0.15.0/docker-compose.yml
index 956602f46..32dac09f2 100644
--- a/devops/releases/lake-v0.15.0/docker-compose.yml
+++ b/devops/releases/lake-v0.15.0/docker-compose.yml
@@ -27,6 +27,10 @@ services:
MYSQL_DATABASE: lake
MYSQL_USER: merico
MYSQL_PASSWORD: merico
+ command:
+ --character-set-server=utf8mb4
+ --collation-server=utf8mb4_bin
+
grafana:
image: apache/devlake-dashboard:v0.15.0
diff --git a/docker-compose.yml b/docker-compose.yml
index 1183b17c5..5fe61e7a0 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -27,6 +27,10 @@ services:
MYSQL_DATABASE: lake
MYSQL_USER: merico
MYSQL_PASSWORD: merico
+ command:
+ --character-set-server=utf8mb4
+ --collation-server=utf8mb4_bin
+
# postgres:
# image: postgres:14.2
