This is an automated email from the ASF dual-hosted git repository.
lynwee pushed a commit to branch dev-1
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/dev-1 by this push:
new 708addb0b feat(linker): add e2e test, fix dora errors
708addb0b is described below
commit 708addb0b1646c5c12aa64172a4b874285df4a98
Author: d4x1 <[email protected]>
AuthorDate: Fri May 24 16:39:59 2024 +0800
feat(linker): add e2e test, fix dora errors
---
backend/core/plugin/hub.go | 8 ++-
backend/core/runner/loader.go | 20 ++++--
backend/plugins/dora/impl/impl.go | 9 ++-
.../plugins/linker/e2e/link_pr_and_issue_test.go | 74 ++++++++++++++++++++++
.../plugins/linker/e2e/snapshot_tables/issues.csv | 2 +
.../linker/e2e/snapshot_tables/project_mapping.csv | 2 +
.../e2e/snapshot_tables/pull_request_issues.csv | 2 +
.../linker/e2e/snapshot_tables/pull_requests.csv | 2 +
8 files changed, 109 insertions(+), 10 deletions(-)
diff --git a/backend/core/plugin/hub.go b/backend/core/plugin/hub.go
index 6788c9057..3feca2219 100644
--- a/backend/core/plugin/hub.go
+++ b/backend/core/plugin/hub.go
@@ -20,6 +20,7 @@ package plugin
import (
"fmt"
"strings"
+ "sync"
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
@@ -27,9 +28,14 @@ import (
// Allowing plugin to know each other
-var plugins map[string]PluginMeta
+var (
+ plugins map[string]PluginMeta
+ pluginMutex sync.RWMutex
+)
func RegisterPlugin(name string, plugin PluginMeta) errors.Error {
+ pluginMutex.Lock()
+ defer pluginMutex.Unlock()
if plugins == nil {
plugins = make(map[string]PluginMeta)
}
diff --git a/backend/core/runner/loader.go b/backend/core/runner/loader.go
index 6e435a172..c454aeceb 100644
--- a/backend/core/runner/loader.go
+++ b/backend/core/runner/loader.go
@@ -23,6 +23,7 @@ import (
"path/filepath"
goplugin "plugin"
"strings"
+ "sync"
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
@@ -49,6 +50,7 @@ func LoadPlugins(basicRes context.BasicRes) errors.Error {
func LoadGoPlugins(basicRes context.BasicRes) errors.Error {
pluginsDir := basicRes.GetConfig("PLUGIN_DIR")
+ var wg sync.WaitGroup
walkErr := filepath.WalkDir(pluginsDir, func(path string, d
fs.DirEntry, err error) error {
if err != nil {
return err
@@ -68,15 +70,21 @@ func LoadGoPlugins(basicRes context.BasicRes) errors.Error {
if !ok {
return errors.Default.New(fmt.Sprintf("%s
PluginEntry must implement PluginMeta interface", pluginName))
}
- err = plugin.RegisterPlugin(pluginName, pluginMeta)
- if err != nil {
- return err
- }
-
- basicRes.GetLogger().Info(`plugin loaded %s`,
pluginName)
+ wg.Add(1)
+ go func(pluginName string, pluginMeta
plugin.PluginMeta) {
+ defer func() {
+ wg.Done()
+ }()
+ err = plugin.RegisterPlugin(pluginName,
pluginMeta)
+ if err != nil {
+ panic(err)
+ }
+ basicRes.GetLogger().Info(`plugin loaded %s`,
pluginName)
+ }(pluginName, pluginMeta)
}
return nil
})
+ wg.Wait()
return errors.Convert(walkErr)
}
diff --git a/backend/plugins/dora/impl/impl.go
b/backend/plugins/dora/impl/impl.go
index b20ce189d..f96a04b5b 100644
--- a/backend/plugins/dora/impl/impl.go
+++ b/backend/plugins/dora/impl/impl.go
@@ -119,10 +119,13 @@ func (p Dora) MigrationScripts() []plugin.MigrationScript
{
func (p Dora) MakeMetricPluginPipelinePlanV200(projectName string, options
json.RawMessage) (coreModels.PipelinePlan, errors.Error) {
op := &tasks.DoraOptions{}
- err := json.Unmarshal(options, op)
- if err != nil {
- return nil, errors.Default.WrapRaw(err)
+ if options != nil && string(options) != "\"\"" {
+ err := json.Unmarshal(options, op)
+ if err != nil {
+ return nil, errors.Default.WrapRaw(err)
+ }
}
+
plan := coreModels.PipelinePlan{
{
{
diff --git a/backend/plugins/linker/e2e/link_pr_and_issue_test.go
b/backend/plugins/linker/e2e/link_pr_and_issue_test.go
new file mode 100644
index 000000000..ad2ae7eec
--- /dev/null
+++ b/backend/plugins/linker/e2e/link_pr_and_issue_test.go
@@ -0,0 +1,74 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "regexp"
+ "testing"
+
+ "github.com/apache/incubator-devlake/core/models/domainlayer/code"
+
"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/plugins/linker/impl"
+ "github.com/apache/incubator-devlake/plugins/linker/tasks"
+)
+
+func TestLinkPrToIssue(t *testing.T) {
+ var plugin impl.Linker
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "issue_linker", plugin)
+
+ regexpStr := "#(\\d+)"
+ re, err := regexp.Compile(regexpStr)
+ if err != nil {
+ panic(err)
+ }
+ taskData := &tasks.LinkerTaskData{
+ Options: &tasks.LinkerOptions{
+ //
https://docs.gitlab.com/ee/user/project/issues/crosslinking_issues.html
+ // For gitlab issues #xxx, GL-xxxx, projectname#xxx or
https://gitlab.com/<username>/<projectname>/-/issues/<xxx>
+ // https://regex101.com/r/RteyFk/1
+ PrToIssueRegexp: regexpStr,
+ ProjectName: "GitHub1",
+ },
+ PrToIssueRegexp: re,
+ }
+
+ dataflowTester.ImportCsvIntoTabler("./snapshot_tables/issues.csv",
&ticket.Issue{})
+
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/pull_requests.csv",
&code.PullRequest{})
+
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/project_mapping.csv",
&crossdomain.ProjectMapping{})
+
+ // verify extraction
+ dataflowTester.FlushTabler(&crossdomain.PullRequestIssue{})
+ dataflowTester.Subtask(tasks.LinkPrToIssueMeta, taskData)
+ dataflowTester.VerifyTable(
+ crossdomain.PullRequestIssue{},
+ "./snapshot_tables/pull_request_issues.csv",
+ []string{
+ "pull_request_id",
+ "pull_request_key",
+ "issue_id",
+ "issue_key",
+ "_raw_data_params",
+ "_raw_data_table",
+ "_raw_data_id",
+ "_raw_data_remark",
+ },
+ )
+
+}
diff --git a/backend/plugins/linker/e2e/snapshot_tables/issues.csv
b/backend/plugins/linker/e2e/snapshot_tables/issues.csv
new file mode 100644
index 000000000..a7d1fd0f9
--- /dev/null
+++ b/backend/plugins/linker/e2e/snapshot_tables/issues.csv
@@ -0,0 +1,2 @@
+"id","created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark","url","icon_url","issue_key","title","description","epic_key","type","original_type","status","original_status","resolution_date","created_date","updated_date","lead_time_minutes","parent_issue_id","priority","story_point","original_estimate_minutes","time_spent_minutes","time_remaining_minutes","creator_id","creator_name","assignee_id","assignee_name","severity","component","original_pr
[...]
+"github:GithubIssue:1:1237324696","2024-05-14 10:42:37.529","2024-05-15
12:07:36.450","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_graphql_issues",59,"","https://github.com/apache/incubator-devlake/issues/1884","","1884","Add
a plugin for
Ones","desc","","","type/feature-request,Stale,add-a-plugin","TODO","OPEN","2032-05-16
15:23:21.000","2022-05-16 15:23:21.000","2024-05-11
00:17:21.000",10,"","",11,1,12,11,"github:GithubAccount:1:14050754","Startrekzky","",
[...]
diff --git a/backend/plugins/linker/e2e/snapshot_tables/project_mapping.csv
b/backend/plugins/linker/e2e/snapshot_tables/project_mapping.csv
new file mode 100644
index 000000000..70e57627e
--- /dev/null
+++ b/backend/plugins/linker/e2e/snapshot_tables/project_mapping.csv
@@ -0,0 +1,2 @@
+"project_name","table","row_id","created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark"
+"GitHub1","cicd_scopes","github:GithubRepo:1:384111310","2024-05-15
12:02:13.590","2024-05-15 12:02:13.590","GitHub1","",0,""
\ No newline at end of file
diff --git a/backend/plugins/linker/e2e/snapshot_tables/pull_request_issues.csv
b/backend/plugins/linker/e2e/snapshot_tables/pull_request_issues.csv
new file mode 100644
index 000000000..4f952b6fd
--- /dev/null
+++ b/backend/plugins/linker/e2e/snapshot_tables/pull_request_issues.csv
@@ -0,0 +1,2 @@
+pull_request_id,issue_id,pull_request_key,issue_key,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+github:GithubPullRequest:1:1819250573,github:GithubIssue:1:1237324696,7317,1884,,,0,"pull_requests,"
diff --git a/backend/plugins/linker/e2e/snapshot_tables/pull_requests.csv
b/backend/plugins/linker/e2e/snapshot_tables/pull_requests.csv
new file mode 100644
index 000000000..bf260bf0c
--- /dev/null
+++ b/backend/plugins/linker/e2e/snapshot_tables/pull_requests.csv
@@ -0,0 +1,2 @@
+"id","created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark","base_repo_id","base_ref","base_commit_sha","head_repo_id","head_ref","head_commit_sha","merge_commit_sha","status","original_status","type","component","title","description","url","author_name","author_id","parent_pr_id","pull_request_key","created_date","merged_date","closed_date"
+"github:GithubPullRequest:1:1819250573","2024-05-15 12:07:36.778","2024-05-15
12:07:36.778","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_api_pull_requests",191,"","github:GithubRepo:1:384111310","main","64c52748f3529784cb6c8a372691aa0f638fa73d","github:GithubRepo:1:384111310","fix#7275","14fb6488f2208e6a65374a86efce12dd460987e0","91dbce48759da14a4a030124c3ef751f1c5d8389","CLOSED","closed","","","fix:
can't GET projects which have / in their name #1884","desc" [...]
\ No newline at end of file
